parallel.cpp 205 KB
Newer Older
Kirill Terekhov's avatar
Kirill Terekhov committed
1 2 3 4
#ifdef _MSC_VER //kill some warnings
#define _SCL_SECURE_NO_WARNINGS
#endif

Kirill Terekhov's avatar
Kirill Terekhov committed
5 6 7 8 9 10 11
#include "inmost.h"


#if defined(USE_MESH)
#include <iostream>
#include <fstream>
#include <sstream>
12
#include <string>
Kirill Terekhov's avatar
Kirill Terekhov committed
13

14
using namespace std;
Kirill Terekhov's avatar
Kirill Terekhov committed
15 16 17 18 19 20 21 22 23

#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
#endif

#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
24 25 26
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
27 28
#else
#define REPORT_MPI(x) x
Kirill Terekhov's avatar
Kirill Terekhov committed
29 30 31 32 33
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE()  {}
Kirill Terekhov's avatar
Kirill Terekhov committed
34 35 36 37
#endif

namespace INMOST
{
38 39 40 41 42 43 44 45 46 47 48 49
    static int flag = 0;

    std::string ro()
    {
        int rank = 0;
        MPI_Comm_rank(MPI_COMM_WORLD,&rank);
        std::stringstream ss;
        for (int i = 0; i < rank; i++)
            ss << "   ";
        return ss.str();
    }
    
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
	//////////////////////////////
	/// REDUCTION FUNCTIONS    ///
	//////////////////////////////
	void DefaultUnpack(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size == 0 )
		{
			if( tag.isDefined(element->GetElementType()) ) 
			{
				if( tag.isSparse(element->GetElementType()) )
				{
					if( element->HaveData(tag) )
						element->DelData(tag); 
				}
				else if( tag.GetSize() == ENUMUNDEF )
					element->SetDataSize(tag,size);
				else
				{
					std::cerr << "received zero size for dense nonzero tag" << std::endl;
					assert(false);
				}
			}
			return;
		}
		if( !element->HaveData(tag) )
			element->SetDataSize(tag,size);
		else if( size != element->GetDataSize(tag) )
		{
			if( tag.GetSize() == ENUMUNDEF )
				element->SetDataSize(tag,size);
			else
			{
				assert(false);
			}
		}
		element->SetData(tag,0,size,data);
	}


	void UnpackOnSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
93
      const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
94 95 96 97 98 99 100 101 102 103
			Storage::integer_array arr = e->IntegerArray(tag);
			arr.push_back(recv[0]);
		}
	}


	void UnpackSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
      for(INMOST_DATA_ENUM_TYPE k = 0; k < size; k+=2 )
      {
			  bool flag = true;
			  const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data))+k;
			  Storage::integer_array arr = e->IntegerArray(tag);
      
        for(Storage::integer_array::iterator it = arr.begin(); it != arr.end(); it+=2)
        {
          if( *it == recv[0] )
				  {
					  flag = false;
					  break;
				  }
        }
			  if( flag ) 
			  {
				  arr.push_back(recv[0]);
				  arr.push_back(recv[1]);
			  }
      }
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
		}
	}
	


	void DeleteUnpack(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
	void RedistUnpack(const Tag & tag,const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			Storage::integer_array p1 = e->IntegerArray(tag);
			const Storage::integer * p2 = static_cast<const Storage::integer *>(static_cast<const void *>(data));
			dynarray<Storage::integer,64> result(p1.size()+size);
			dynarray<Storage::integer,64>::iterator end;
			end = std::set_union(p1.begin(),p1.end(),p2,p2+size,result.begin());
			result.resize(end-result.begin());
			p1.clear();
			p1.insert(p1.end(),result.begin(),result.end());
		}
	}

	void UnpackLayersMarker(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
168

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
169
	void UnpackSyncMarkerOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
170
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
171
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
172 173 174
		element->Bulk(tag) |= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
175
	void UnpackSyncMarkerXOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
176
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
177
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
178 179 180 181
		element->Bulk(tag) ^= *data;
	}

	
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
182
	void UnpackSyncMarkerAND(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
183
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
184
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
185 186 187
		element->Bulk(tag) &= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
188 189 190
	//////////////////////////////
	/// REDUCTION FUNCTIONS END///
	//////////////////////////////
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
	bool Mesh::HaveGlobalID(ElementType type) const
	{
		assert(OneType(type));
		Tag test;
		if( GlobalIDTag().isValid() ) test = GlobalIDTag();
		//else if( HaveTag("GLOBAL_ID") ) test = GetTag("GLOBAL_ID") );
		if( test.isValid() )
		{
			//bool ret = true;
			//for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype) ) if( etype & types )
			//	ret &= test.isDefined(etype);
			//return ret;
			return test.isDefined(type);
		}
		else return false;
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
207

Kirill Terekhov's avatar
Kirill Terekhov committed
208

Kirill Terekhov's avatar
Kirill Terekhov committed
209
	void Mesh::SynchronizeMarker(MarkerType marker, ElementType mask, SyncBitOp op)
Kirill Terekhov's avatar
Kirill Terekhov committed
210
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
211
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
212 213 214 215
#if defined(USE_MPI)
		if( m_state == Mesh::Parallel )
		{
			Tag t = CreateTag("TEMP_SYNC_MARKER",DATA_BULK,mask,mask,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
216
			
Kirill Terekhov's avatar
Kirill Terekhov committed
217 218 219
			//workaround for old gcc compiler
			const Element::Status SGhost = Element::Ghost;
			const Element::Status SAny = Element::Any;
Kirill Terekhov's avatar
Kirill Terekhov committed
220
			Element::Status Expr = (Element::Shared | ((op != SYNC_BIT_SET) ? SGhost : SAny));
Kirill Terekhov's avatar
Kirill Terekhov committed
221
			
Kirill Terekhov's avatar
Kirill Terekhov committed
222
			for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
223
				if( it->GetMarker(marker) && (it->GetStatus() & Expr) )
Kirill Terekhov's avatar
Kirill Terekhov committed
224
					it->Bulk(t) = 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
225
			
Kirill Terekhov's avatar
Kirill Terekhov committed
226 227 228 229
			
			
			switch(op)
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
230 231 232
				case SYNC_BIT_SET:
					ExchangeData(t,mask,0);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
233
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
234 235 236 237
						if( it->GetStatus() == Element::Ghost )
						{
							if( it->HaveData(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
238
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
239 240 241 242
					break;
				case SYNC_BIT_OR:
					ReduceData(t,mask,0,UnpackSyncMarkerOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
243
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
244 245 246 247
						if( it->GetStatus() & (Element::Ghost | Element::Shared) )
						{
							if( !it->GetMarker(marker) && it->HaveData(t) ) it->SetMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
248
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
249 250 251 252
					break;
				case SYNC_BIT_AND:
					ReduceData(t,mask,0,UnpackSyncMarkerAND);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
253
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
254 255 256 257
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
258
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
259 260 261 262
					break;
				case SYNC_BIT_XOR:
					ReduceData(t,mask,0,UnpackSyncMarkerXOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
263
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
264 265 266 267
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
268
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
269
					break;
Kirill Terekhov's avatar
Kirill Terekhov committed
270 271
			}
			
Kirill Terekhov's avatar
Kirill Terekhov committed
272
			
Kirill Terekhov's avatar
Kirill Terekhov committed
273 274
			DeleteTag(t,mask);
		}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
275 276 277 278 279
#else//USE_MPI
		(void) marker;
		(void) mask;
		(void) op;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
280
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
281
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
282
	
Kirill Terekhov's avatar
Kirill Terekhov committed
283 284
	ElementType Mesh::SynchronizeElementType(ElementType etype)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
285
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
286 287
		ElementType etypeout = etype;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
288
		REPORT_MPI(MPI_Allreduce(&etype,&etypeout,1,INMOST_MPI_DATA_BULK_TYPE,MPI_BOR,GetCommunicator()));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
289
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
290 291
		return etypeout;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
292 293
	Storage::integer Mesh::TotalNumberOf(ElementType mask)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
294
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
295 296 297 298 299
		Storage::integer number = 0, ret = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
			if( it->GetStatus() != Element::Ghost ) number++;
		ret = number;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
300
		REPORT_MPI(MPI_Allreduce(&number,&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
301
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
302
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
303 304
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
305
	Storage::integer Mesh::EnumerateSet(const ElementSet & set, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
306
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
307
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
308 309 310 311
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
312 313
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
314
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
315
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
316
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
317
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
318
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
319 320
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
321
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
322 323
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
324
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
325
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
326 327
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
328
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
329
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
330
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
331
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
332 333
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
334
	Storage::integer Mesh::Enumerate(const HandleType * set, enumerator n, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
335
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
336
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
337 338 339 340
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
341 342
		for(const HandleType * it = set; it != set+n; ++it)
			if( GetStatus(*it) != Element::Ghost && (define_sparse || HaveData(*it,num_tag))) number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
343
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
344
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
345
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
346
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
347
		for(const HandleType * it = set; it != set+n; ++it)
348
			if( GetStatus(*it) == Element::Owned && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
349
		for(const HandleType * it = set; it != set+n; ++it)
350
			if( GetStatus(*it) == Element::Shared && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
351
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
352 353
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
354
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
355
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
356
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
357
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
358 359
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
360
	Storage::integer Mesh::Enumerate(ElementType mask, Tag num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
361
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
362
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
363 364 365 366
		Storage::integer shift = 0, ret = 0;
#if defined(USE_MPI)
		Storage::integer number = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
367
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
368
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
369
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
370
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
371
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
372 373
		shift += start;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
374
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
375 376
				it->Integer(num_tag) = shift++;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
377
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
378
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
379
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
380 381
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
382
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
383
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
384
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
385
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
386 387 388 389 390
		return ret;
	}
	
	Storage::real Mesh::Integrate(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
391
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
392 393
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
394
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
395 396 397
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
398
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
399 400
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
401 402
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
403
	
Kirill Terekhov's avatar
Kirill Terekhov committed
404 405 406
	
	Storage::integer Mesh::Integrate(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
407
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
408 409
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
410
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
411 412 413
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
414
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
415 416
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
417
	
Kirill Terekhov's avatar
Kirill Terekhov committed
418 419
	void Mesh::Integrate(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
420
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
421
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
422 423
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
424
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
425
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
426
#else//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
427 428
		(void) input;
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
429
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
430
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
431
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
432
	
Kirill Terekhov's avatar
Kirill Terekhov committed
433 434
	void Mesh::Integrate(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
435
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
436
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
437 438
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
439
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
440
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
441 442
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
443
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
444
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
445
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
446
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
447 448 449
	
	Storage::integer Mesh::ExclusiveSum(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
450
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
451 452
		Storage::integer output = 0;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
453
		REPORT_MPI(MPI_Scan(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
454
		output -= input;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
455 456 457
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
458
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
459 460 461
		return output;
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
462
	Storage::real Mesh::Integrate(const Tag & t, enumerator entry, ElementType mask)
Kirill Terekhov's avatar
Kirill Terekhov committed
463
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
464
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
465 466
		Storage::real output = 0, input = 0;
		for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
467
			if( GetStatus(*it) != Element::Ghost && HaveData(*it,t) )
Kirill Terekhov's avatar
Kirill Terekhov committed
468 469 470 471
			{
				real_array arr = RealArray(*it,t);
				if( arr.size() > entry ) input += arr[entry];
			}
Kirill Terekhov's avatar
Kirill Terekhov committed
472 473
		output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
474
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
475
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
476
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
477 478 479
		return output;
	}
	
480 481
	Storage::real Mesh::AggregateMax(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
482
		ENTER_FUNC();
483 484
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
485
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
486 487 488
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
489
		EXIT_FUNC();
490 491
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
492
	
493 494
	Storage::integer Mesh::AggregateMax(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
495
		ENTER_FUNC();
496 497
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
498
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
499 500 501
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
502
		EXIT_FUNC();
503 504
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
505
	
Kirill Terekhov's avatar
Kirill Terekhov committed
506 507
	void Mesh::AggregateMax(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
508
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
509
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
510 511
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
512
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
513
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
514 515
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
516
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
517
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
518
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
519
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
520
	
Kirill Terekhov's avatar
Kirill Terekhov committed
521 522
	void Mesh::AggregateMax(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
523
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
524
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
525 526
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
527
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
528
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
529 530
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
531
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
532
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
533
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
534
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
535
	
Kirill Terekhov's avatar
Kirill Terekhov committed
536 537
	Storage::real Mesh::AggregateMin(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
538
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
539 540
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
541
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
542 543 544
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
545
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
546 547 548 549 550
		return output;
	}
	
	Storage::integer Mesh::AggregateMin(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
551
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
552 553
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
554
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
555 556 557
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
558
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
559 560
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
561
	
Kirill Terekhov's avatar
Kirill Terekhov committed
562 563
	void Mesh::AggregateMin(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
564
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
565
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
566 567
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
568
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
569
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
570 571
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
572
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
573
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
574
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
575
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
576
	
Kirill Terekhov's avatar
Kirill Terekhov committed
577 578
	void Mesh::AggregateMin(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
579
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
580
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
581 582
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
583
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
584
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
585 586
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
587
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
588
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
589
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
590
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
591 592
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
593
	INMOST_MPI_Comm Mesh::GetCommunicator() const
Kirill Terekhov's avatar
Kirill Terekhov committed
594 595 596
	{
#if defined(USE_MPI)
		return comm;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
597
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
598
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
599
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
600 601
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
602
	int Mesh::GetProcessorRank() const
Kirill Terekhov's avatar
Kirill Terekhov committed
603 604 605 606 607
	{
#if defined(USE_MPI)
		int rank;
		MPI_Comm_rank(comm,&rank);
		return rank;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
608
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
609
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
610
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
611 612
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
613
	int Mesh::GetProcessorsNumber() const
Kirill Terekhov's avatar
Kirill Terekhov committed
614 615 616 617 618
	{
#if defined(USE_MPI)
		int size;
		MPI_Comm_size(comm,&size);
		return size;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
619
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
620
		return 1;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
621
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
622 623 624 625 626 627 628 629
	}	
	
	void Mesh::Initialize(int * argc, char *** argv)
	{
#if defined(USE_MPI)
		int test;
		MPI_Initialized(&test);
        if( test == 0 ) MPI_Init(argc,argv);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
630 631 632 633
#else //USE_MPI
		(void) argc;
		(void) argv;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
634 635 636
#if defined(USE_PARALLEL_WRITE_TIME)
		atexit(Mesh::AtExit);
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
637 638 639 640 641 642 643 644
	}
	void Mesh::Finalize()
	{
#if defined(USE_MPI)
		int test = 0;
		MPI_Finalized(&test);
		if( !test )
			MPI_Finalize();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
645
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
646 647
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
648
    INMOST_MPI_Group Mesh::GetGroup() const
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
649 650 651 652 653 654 655
    {
        INMOST_MPI_Group ret = INMOST_MPI_GROUP_EMPTY;
#if defined(USE_MPI)
        MPI_Comm_group(GetCommunicator(), &ret);
#endif
        return ret;
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
656 657 658 659
	
	void Mesh::SetCommunicator(INMOST_MPI_Comm _comm)
	{
		ENTER_FUNC();
660 661 662
		tag_shared = CreateTag("PROTECTED_STATUS",DATA_BULK,  ESET |CELL | FACE | EDGE | NODE,NONE,1);
		tag_owner = CreateTag("OWNER_PROCESSOR",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE,NONE,1);
		tag_processors = CreateTag("PROCESSORS_LIST",DATA_INTEGER, ESET|  MESH | NODE | EDGE | FACE | CELL,NONE);
Kirill Terekhov's avatar
Kirill Terekhov committed
663 664
		tag_layers = CreateTag("LAYERS",DATA_INTEGER,MESH,NONE,1);
		tag_bridge = CreateTag("BRIDGE",DATA_INTEGER,MESH,NONE,1);
665 666
		tag_sendto = CreateTag("PROTECTED_SENDTO",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE, ESET | CELL | FACE | EDGE | NODE);
		tag_an_id = CreateTag("PROTECTED_ID",DATA_INTEGER, CELL | FACE | EDGE | NODE, CELL | FACE | EDGE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
667 668

#if defined(USE_MPI)
669
		randomizer = Random();
Kirill Terekhov's avatar
Kirill Terekhov committed
670 671 672 673 674 675 676 677 678 679 680
		
		parallel_strategy = 1;
		parallel_file_strategy = 1;

		

		Mesh::Initialize(NULL,NULL);
		//~ MPI_Comm_dup(_comm,&comm);
		comm = _comm;
		{
			INMOST_DATA_BIG_ENUM_TYPE t = pmid;
681
			REPORT_MPI(MPI_Allreduce(&t,&parallel_mesh_unique_id,1,INMOST_MPI_DATA_BIG_ENUM_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
682 683 684 685 686
			pmid = parallel_mesh_unique_id+1;
		}
		m_state = Mesh::Parallel;


Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
687
#if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705
		int err;
		REPORT_MPI(err = MPI_Alloc_mem(GetProcessorsNumber()*sizeof(unsigned)*2,MPI_INFO_NULL,&shared_space));
		if( err )
		{
			int errclass;
			MPI_Error_class(err,&errclass);
			std::cout << "Cannot allocate shared space of size " << GetProcessorsNumber()*sizeof(unsigned)*2 << " reason is "; 
			switch(err)
			{
				case MPI_SUCCESS: std::cout << "success"; break;
				case MPI_ERR_INFO: std::cout << "bad info"; break;
				case MPI_ERR_ARG: std::cout << "bad argument"; break;
				case MPI_ERR_NO_MEM: std::cout << "no memory"; break;
			}
			std::cout << std::endl;
			MPI_Abort(comm,err);
		}
		REPORT_MPI(MPI_Win_create(shared_space,sizeof(unsigned)*GetProcessorsNumber()*2,sizeof(unsigned),MPI_INFO_NULL,comm,&window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
706 707 708 709
#endif //USE_MPI_P2P
#else //USE_MPI
		(void) _comm;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
710 711 712 713 714 715
		EXIT_FUNC();
	}
	
#if defined(USE_PARALLEL_WRITE_TIME)
	void Mesh::Enter() { tab++; }
	void Mesh::Exit() {tab--; }
Kirill Terekhov's avatar
Kirill Terekhov committed
716
	std::ostream & Mesh::WriteTab(std::ostream & f)
Kirill Terekhov's avatar
Kirill Terekhov committed
717 718 719 720 721 722 723 724 725
	{
		for(int i = 0; i < tab; i++)
			f << "   ";
		return f;
	}
	std::fstream & Mesh::GetStream()
	{
		return out_time;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
726 727
	void Mesh::FinalizeFile()
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
728 729
    //std::stringstream str;
    if( tab > 1 )
Kirill Terekhov's avatar
Kirill Terekhov committed
730
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
731
      out_time << "<TEXT><![CDATA[Died!]]></TEXT>\n";// << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
732
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
733
    while(tab > 1)
Kirill Terekhov's avatar
Kirill Terekhov committed
734
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
735 736
      out_time << "<TIME>-1</TIME>\n</FUNCTION>\n";// << std::endl; 
      Exit(); 
Kirill Terekhov's avatar
Kirill Terekhov committed
737 738
		}
		out_time << "</Debug>" << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
739 740 741
    //out_time << str;
    //out_time.flush();
    out_time.close();
Kirill Terekhov's avatar
Kirill Terekhov committed
742
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
743
#endif //USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
744 745 746
	
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
747
	void determine_my_procs_low(Mesh * m, HandleType h, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
748
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
749 750 751
		Element::adj_type const & subelements = m->LowConn(h);
		Element::adj_type::const_iterator i = subelements.begin();
		Storage::integer_array p = m->IntegerArrayDV(*i,m->ProcessorsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
752 753 754 755 756
		result.clear();
		result.insert(result.end(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
757 758
			Storage::integer_array q = m->IntegerArrayDV(*i,m->ProcessorsTag());
			intersection.resize(std::max(static_cast<unsigned>(result.size()),static_cast<unsigned>(q.size())));
Kirill Terekhov's avatar
Kirill Terekhov committed
759 760 761 762 763 764 765
			dynarray<Storage::integer,64>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
766
	void determine_my_procs_high(Mesh * m, HandleType h, const Tag & procs, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
767
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
768
		Element::adj_type const & overelements = m->HighConn(h);
Kirill Terekhov's avatar
Kirill Terekhov committed
769
		if( overelements.empty() ) return;
Kirill Terekhov's avatar
Kirill Terekhov committed
770
		Element::adj_type::const_iterator i = overelements.begin();
Kirill Terekhov's avatar
Kirill Terekhov committed
771 772 773
		result.clear();
		while(i != overelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
774
			Storage::integer_array q = m->IntegerArrayDV(*i,procs);
Kirill Terekhov's avatar
Kirill Terekhov committed
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
			intersection.resize(result.size()+q.size());
			dynarray<Storage::integer,64>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
	/*
	void determine_my_procs_low(adjacent<Element> & subelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( subelements.empty() ) return;
		adjacent<Element>::iterator i = subelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size());
			std::vector<Storage::integer>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	
	void determine_my_procs_high(adjacent<Element> & overelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( overelements.empty() ) return;
		adjacent<Element>::iterator i = overelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != overelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size()+q.size());
			std::vector<Storage::integer>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	*/
	/*
	void Mesh::MarkShared(ElementType mask)
	{
		for(ElementType etype = NODE; etype <= CELL; etype = etype << 1 ) if( etype & mask )
		for(Mesh::iteratorElement it = BeginElement(etype); it != EndElement(); it++)
		{
			int owner;
			Storage::integer_array v = it->IntegerArrayDV(tag_processors);
			std::sort(v.begin(),v.end());
			if( v.empty() )
			{
				owner = mpirank;
				v.push_back(mpirank);
			}
			else
				owner = std::min(mpirank,v[0]);
			
			it->IntegerDF(tag_owner) = owner;
			
			if( mpirank == owner )
			{
				if( v.size() == 1 )
					it->BulkDF(tag_shared) = Element::Owned;
				else
					it->BulkDF(tag_shared) = Element::Shared;
			}
			else
				it->BulkDF(tag_shared) = Element::Ghost;
		}
		
		ComputeSharedProcs();
		RecomputeParallelStorage(mask);
		AssignGlobalID(mask);
		
		//have to do it for all types
#if defined(USE_PARALLEL_STORAGE)
		for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
		for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)			
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
#endif
	}
	*/
	
	
871
	__INLINE bool point_in_bbox(Storage::real * p, Storage::real bbox[6], unsigned dim, Storage::real eps)
Kirill Terekhov's avatar
Kirill Terekhov committed
872 873 874
	{
		bool ret = true;
		for(unsigned k = 0; k < dim; k++) 
875
			ret &= (p[k] > bbox[k]-eps && p[k] < bbox[dim+k]+eps);
Kirill Terekhov's avatar
Kirill Terekhov committed
876 877 878 879 880 881
		return ret;
	}
	
	__INLINE bool compare_coord(Storage::real * a, Storage::real * b, INMOST_DATA_ENUM_TYPE dim, Storage::real eps)
	{
		for(INMOST_DATA_ENUM_TYPE i = 0; i <  dim; i++)
Kirill Terekhov's avatar
Kirill Terekhov committed
882
			if( ::fabs(a[i]-b[i]) > eps ) return a[i] <= b[i];
Kirill Terekhov's avatar
Kirill Terekhov committed
883 884
		return true;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
885 886 887 888 889

	class MappingComparator
	{
	public: bool operator () (const std::pair<int,int> & a, const std::pair<int,int> & b) {return a.first < b.first;}
	};
890
    
Kirill Terekhov's avatar
Kirill Terekhov committed
891
	
892 893
	
	void Mesh::ResolveShared(bool only_new)
Kirill Terekhov's avatar
Kirill Terekhov committed
894 895
	{
		ENTER_FUNC();
896
		
Kirill Terekhov's avatar
Kirill Terekhov committed
897 898
#if defined(USE_MPI)
		if( m_state == Mesh::Serial ) SetCommunicator(INMOST_MPI_COMM_WORLD);
899
		if( tag_global_id.isValid() ) tag_global_id = DeleteTag(tag_global_id,CELL | EDGE | FACE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
900
		integer dim = GetDimensions();
Kirill Terekhov's avatar
Kirill Terekhov committed
901 902
		int sendsize;
		int mpirank = GetProcessorRank(),mpisize = GetProcessorsNumber();
903
        int rank = mpirank;
Kirill Terekhov's avatar
Kirill Terekhov committed
904 905 906
#if defined(USE_PARALLEL_STORAGE)
		shared_elements.clear();
		ghost_elements.clear();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
907
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
908 909
		//determine which bboxes i intersect
		dynarray<int,64> procs;
Kirill Terekhov's avatar
Kirill Terekhov committed
910 911 912 913 914
		Storage::real bbox[6]; //local bounding box
		std::vector<Storage::real> bboxs(mpisize*6);
		//Compute local bounding box containing nodes.
		//Will be more convinient to compute (or store)
		//and communicate local octree over all the nodes.
Kirill Terekhov's avatar
Kirill Terekhov committed
915
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
916 917 918 919
		{
			bbox[k] = 1e20;
			bbox[k+dim] = -1e20;
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
920 921 922
#if defined(USE_OMP)
#pragma omp parallel
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
923
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
924
			real bbox0[6];
Kirill Terekhov's avatar
Kirill Terekhov committed
925
			for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
926
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
				bbox0[k] = 1e20;
				bbox0[k+dim] = -1e20;
			}
#if defined(USE_OMP)
#pragma omp for
#endif
			for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
			{
				Node it = NodeByLocalID(nit);
				Storage::real_array arr = it->Coords();
				for(integer k = 0; k < dim; k++)
				{
					if( arr[k] < bbox0[k] ) bbox0[k] = arr[k];
					if( arr[k] > bbox0[k+dim] ) bbox0[k+dim] = arr[k];
				}
			}
#if defined(USE_OMP)
#pragma omp critical
#endif
			{
				for(integer k = 0; k < dim; k++)
				{
					if( bbox0[k] < bbox[k] ) bbox[k] = bbox0[k];
Kirill Terekhov's avatar
Kirill Terekhov committed
950
					if( bbox0[k+dim] > bbox[k+dim] ) bbox[k+dim] = bbox0[k+dim];
Kirill Terekhov's avatar
Kirill Terekhov committed
951
				}
Kirill Terekhov's avatar
Kirill Terekhov committed
952 953
			}
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
954
		REPORT_VAL("dim",dim);
955
		assert(dim*2 <= 6);
Kirill Terekhov's avatar
Kirill Terekhov committed
956
		// write down bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
957
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
958 959 960 961
		{
			REPORT_VAL("min",bbox[k]);
			REPORT_VAL("max",bbox[dim+k]);
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
962
		// communicate bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
963
		REPORT_MPI(MPI_Allgather(&bbox[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,&bboxs[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
964
		// find all processors that i communicate with
Kirill Terekhov's avatar
Kirill Terekhov committed
965 966 967 968
		for(int k = 0; k < mpisize; k++)
			if( k != mpirank )
			{
				bool flag = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
969
				for(integer q = 0; q < dim; q++)
970
					flag &= !((bbox[q]-GetEpsilon() > bboxs[k*dim*2+q+dim]) || (bbox[dim+q]+GetEpsilon() < bboxs[k*dim*2+q]));
Kirill Terekhov's avatar
Kirill Terekhov committed
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
				if( flag ) procs.push_back(k);
			}
		REPORT_VAL("neighbour processors",procs.size());
		
		//~ if( procs.empty() )
		//~ {
			//~ REPORT_STR("no processors around - all elements are owned");
			//~ for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
			//~ {
				//~ it->IntegerArrayDV(tag_processors).resize(1);
				//~ it->IntegerDF(tag_owner) = it->IntegerDV(tag_processors) = mpirank;
				//~ it->BulkDF(tag_shared) = Element::Owned;
			//~ }
			//~ ComputeSharedProcs();
			//~ RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
			//~ AssignGlobalID(CELL | EDGE | FACE | NODE);
		//~ }
		//~ else
		{
			bool same_boxes = true, same_box;
			for(int k = 0; k < mpisize && same_boxes; k++)
			{
				same_box = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
994
				for(integer j = 0; j < dim*2; j++)
995
					same_box &= ::fabs(bbox[j] - bboxs[k*dim*2+j]) < GetEpsilon();
Kirill Terekhov's avatar
Kirill Terekhov committed
996 997 998 999 1000 1001
				same_boxes &= same_box;
			}
			
			if( same_boxes )
			{
				REPORT_STR("All bounding boxes are the same - assuming that mesh is replicated over all nodes");
Kirill Terekhov's avatar
Kirill Terekhov committed
1002 1003
				//for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
				for(ElementType etype = NODE; etype <= CELL; etype = NextElementType(etype) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1004
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1005 1006 1007 1008 1009 1010
#if defined(USE_OMP)
#pragma omp parallel for
#endif
					for(integer eit = 0; eit < LastLocalID(etype); ++eit) if( isValidElement(etype,eit) )
					{
						Element it = ElementByLocalID(etype,eit);
1011 1012
                        if (only_new && GetMarker(it->GetHandle(),NewMarker()) == false) continue;
                        
Kirill Terekhov's avatar
Kirill Terekhov committed
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
						integer_array arr = it->IntegerArrayDV(tag_processors);
						arr.resize(mpisize);
						for(int k = 0; k < mpisize; k++)
							arr[k] = k;
						it->IntegerDF(tag_owner) = 0;
						if( mpirank == 0 )
							SetStatus(it->GetHandle(),Element::Shared);
						else
							SetStatus(it->GetHandle(),Element::Ghost);
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1023 1024 1025 1026 1027 1028 1029 1030 1031
				}
				ComputeSharedProcs();
				RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
				AssignGlobalID(CELL | EDGE | FACE | NODE);
#if defined(USE_PARALLEL_STORAGE)
				for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1032 1033
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1034
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1035
				for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1036 1037 1038
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1039 1040
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1041
					}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1042
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
1043 1044 1045
			}
			else
			{
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
                
                if (only_new)
                {
		            for(Mesh::iteratorCell it = BeginCell(); it != EndCell(); it++) if (GetMarker(*it,NewMarker()))
                    {
                        ElementArray<Node> nodes = it->getNodes();
                        for (int i = 0; i < nodes.size(); i++)
                        {
                            nodes[i].SetMarker(NewMarker());
                        }
                    }
                }

Kirill Terekhov's avatar
Kirill Terekhov committed
1059 1060 1061
				double time = Timer();
				Storage::real epsilon = GetEpsilon();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1062
				GeomParam table;
1063
				for(ElementType etype = EDGE; etype <= CELL; etype = NextElementType(etype))
Kirill Terekhov's avatar
Kirill Terekhov committed
1064 1065 1066 1067 1068 1069 1070 1071 1072
					if( !HaveGeometricData(CENTROID,etype) )
						table[CENTROID] |= etype;
				PrepareGeometricData(table);
				
				time = Timer() - time;
			
				REPORT_STR("Prepare geometric data");
				REPORT_VAL("time",time);
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1073 1074 1075 1076 1077
				//for(iteratorNode it = BeginNode(); it != EndNode(); it++)
#if defined(USE_OMP)
#pragma omp parallel for
#endif
				for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1078
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1079
					Node it = NodeByLocalID(nit);
Kirill Terekhov's avatar
Kirill Terekhov committed
1080 1081 1082 1083 1084 1085
					Storage::integer_array arr = it->IntegerArrayDV(tag_processors);
					arr.resize(1);
					arr[0] = mpirank;
				}
			

Kirill Terekhov's avatar
Kirill Terekhov committed
1086
				buffer_type exch_data;
Kirill Terekhov's avatar
Kirill Terekhov committed
1087 1088
				std::vector< INMOST_DATA_REAL_TYPE > unpack_real;
				std::vector< INMOST_DATA_REAL_TYPE > pack_real;
Kirill Terekhov's avatar
Kirill Terekhov committed
1089
				element_set sorted_nodes;
Kirill Terekhov's avatar
Kirill Terekhov committed
1090 1091 1092 1093 1094 1095 1096 1097 1098
				
				sorted_nodes.reserve(NumberOfNodes());
				
				
				time = Timer();
				
				
				for(iteratorNode n = BeginNode(); n != EndNode(); n++)
				{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1099 1100
					real_array c = n->Coords();
					for(real_array::size_type k = 0; k < procs.size(); k++)
1101
						if( point_in_bbox(c.data(),bboxs.data()+procs[k]*dim*2,dim,GetEpsilon()) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1102
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1103
							sorted_nodes.push_back(*n);
Kirill Terekhov's avatar
Kirill Terekhov committed
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
							break;
						}
				}
				
				time = Timer() - time;
				REPORT_STR("Prepare array of nodes");
				REPORT_VAL("time",time);
				REPORT_VAL("share nodes", sorted_nodes.size());
				REPORT_VAL("total nodes", NumberOfNodes());
				
				
				
				time = Timer();
				if( !sorted_nodes.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1118 1119
					std::sort(sorted_nodes.begin(),sorted_nodes.end(),CentroidComparator(this));
					//qsort(&sorted_nodes[0],sorted_nodes.size(),sizeof(Element *),CompareElementsCCentroid);
Kirill Terekhov's avatar
Kirill Terekhov committed
1120 1121 1122 1123 1124 1125 1126 1127 1128
				time = Timer() - time;
				REPORT_STR("Sort nodes");
				REPORT_VAL("time",time);
				
				
				pack_real.reserve(sorted_nodes.size()*dim);
				
				
				time = Timer();
Kirill Terekhov's avatar
Kirill Terekhov committed
1129
				for(element_set::iterator it = sorted_nodes.begin(); it != sorted_nodes.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1130
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1131
					Storage::real_array arr = RealArrayDF(*it,CoordsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
1132 1133 1134 1135 1136 1137 1138 1139 1140
					pack_real.insert(pack_real.end(),arr.begin(),arr.end());
				}
				time = Timer() - time;
				REPORT_STR("Gather coordinates");
				REPORT_VAL("time",time);
				
				
				time = Timer();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1141
				MPI_Pack_size(static_cast<int>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm,&sendsize);
Kirill Terekhov's avatar
Kirill Terekhov committed
1142 1143
				exch_data.resize(sendsize);
				int position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1144
				if( sendsize > 0 ) MPI_Pack(&pack_real[0],static_cast<INMOST_MPI_SIZE>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,&exch_data[0],static_cast<INMOST_MPI_SIZE>(exch_data.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1145 1146 1147 1148 1149 1150 1151
				
				
				time = Timer() - time;
				REPORT_STR("Pack coordinates");
				REPORT_VAL("time",time);

				
1152
				
Kirill Terekhov's avatar
Kirill Terekhov committed
1153 1154 1155 1156
				{
					std::vector< MPI_Request > send_reqs, recv_reqs;
					exch_buffer_type send_buffs(procs.size()), recv_buffs(procs.size());
					std::vector<int> done;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1157
//~ #if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
1158 1159
					//~ unsigned * sendsizeall = shared_space;
					//~ unsigned usend[2] = {sendsize,pack_real.size()};
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1160
					//~ REPORT_MPI(MPI_Win_fence(0,window)); //start exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1161 1162
					//~ for(unsigned k = 0; k < procs.size(); k++)
						//~ REPORT_MPI(MPI_Put(usend,2,MPI_UNSIGNED,procs[k],mpirank*2,2,MPI_UNSIGNED,window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1163
					//~ REPORT_MPI(MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED,window)); //end exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1164 1165 1166
//~ #else
					std::vector<unsigned> sendsizeall(mpisize*2);
					int pack_size2 = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1167
					unsigned usend[2] = {static_cast<unsigned>(sendsize),static_cast<unsigned>(pack_real.size())};
Kirill Terekhov's avatar
Kirill Terekhov committed
1168
					MPI_Pack_size(2,MPI_UNSIGNED,comm,&pack_size2);
Kirill Terekhov's avatar
Kirill Terekhov committed
1169
					for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1170 1171 1172 1173
					{
						send_buffs[k].first = procs[k];
						send_buffs[k].second.resize(pack_size2);
						position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1174
						MPI_Pack(usend,2,MPI_UNSIGNED,&send_buffs[k].second[0],static_cast<INMOST_MPI_SIZE>(send_buffs[k].second.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1175 1176 1177 1178 1179 1180 1181 1182 1183
						recv_buffs[k].first = procs[k];
						recv_buffs[k].second.resize(pack_size2);
					}
					ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
					while( !(done = FinishRequests(recv_reqs)).empty() )
					{
						for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
						{
							position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1184
							MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&sendsizeall[procs[*qt]*2],2,MPI_UNSIGNED,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1185 1186 1187
						}
					}
					if( !send_reqs.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1188
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
1189
						REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
Kirill Terekhov's avatar
Kirill Terekhov committed
1190
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1191 1192 1193 1194 1195 1196
					//~ REPORT_MPI(MPI_Allgather(usend,2,MPI_UNSIGNED,&sendsizeall[0],2,MPI_UNSIGNED,comm));
//~ #endif
					double time2 = Timer();
					{
						
						
Kirill Terekhov's avatar