parallel.cpp 176 KB
Newer Older
Kirill Terekhov's avatar
Kirill Terekhov committed
1 2 3 4
#ifdef _MSC_VER //kill some warnings
#define _SCL_SECURE_NO_WARNINGS
#endif

Kirill Terekhov's avatar
Kirill Terekhov committed
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#include "inmost.h"


#if defined(USE_MESH)
#include <iostream>
#include <fstream>
#include <sstream>


#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
#endif

#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
22 23 24
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
25 26
#else
#define REPORT_MPI(x) x
Kirill Terekhov's avatar
Kirill Terekhov committed
27 28 29 30 31
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE()  {}
Kirill Terekhov's avatar
Kirill Terekhov committed
32 33 34 35
#endif

namespace INMOST
{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
	//////////////////////////////
	/// REDUCTION FUNCTIONS    ///
	//////////////////////////////
	void DefaultUnpack(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size == 0 )
		{
			if( tag.isDefined(element->GetElementType()) ) 
			{
				if( tag.isSparse(element->GetElementType()) )
				{
					if( element->HaveData(tag) )
						element->DelData(tag); 
				}
				else if( tag.GetSize() == ENUMUNDEF )
					element->SetDataSize(tag,size);
				else
				{
					std::cerr << "received zero size for dense nonzero tag" << std::endl;
					assert(false);
				}
			}
			return;
		}
		if( !element->HaveData(tag) )
			element->SetDataSize(tag,size);
		else if( size != element->GetDataSize(tag) )
		{
			if( tag.GetSize() == ENUMUNDEF )
				element->SetDataSize(tag,size);
			else
			{
				assert(false);
			}
		}
		element->SetData(tag,0,size,data);
	}


	void UnpackOnSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
79
      const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
80 81 82 83 84 85 86 87 88 89
			Storage::integer_array arr = e->IntegerArray(tag);
			arr.push_back(recv[0]);
		}
	}


	void UnpackSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
      for(INMOST_DATA_ENUM_TYPE k = 0; k < size; k+=2 )
      {
			  bool flag = true;
			  const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data))+k;
			  Storage::integer_array arr = e->IntegerArray(tag);
      
        for(Storage::integer_array::iterator it = arr.begin(); it != arr.end(); it+=2)
        {
          if( *it == recv[0] )
				  {
					  flag = false;
					  break;
				  }
        }
			  if( flag ) 
			  {
				  arr.push_back(recv[0]);
				  arr.push_back(recv[1]);
			  }
      }
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
		}
	}
	


	void DeleteUnpack(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
	void RedistUnpack(const Tag & tag,const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			Storage::integer_array p1 = e->IntegerArray(tag);
			const Storage::integer * p2 = static_cast<const Storage::integer *>(static_cast<const void *>(data));
			dynarray<Storage::integer,64> result(p1.size()+size);
			dynarray<Storage::integer,64>::iterator end;
			end = std::set_union(p1.begin(),p1.end(),p2,p2+size,result.begin());
			result.resize(end-result.begin());
			p1.clear();
			p1.insert(p1.end(),result.begin(),result.end());
		}
	}

	void UnpackLayersMarker(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
154

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
155
	void UnpackSyncMarkerOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
156
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
157
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
158 159 160
		element->Bulk(tag) |= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
161
	void UnpackSyncMarkerXOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
162
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
163
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
164 165 166 167
		element->Bulk(tag) ^= *data;
	}

	
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
168
	void UnpackSyncMarkerAND(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
169
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
170
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
171 172 173
		element->Bulk(tag) &= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
174 175 176
	//////////////////////////////
	/// REDUCTION FUNCTIONS END///
	//////////////////////////////
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
	bool Mesh::HaveGlobalID(ElementType type) const
	{
		assert(OneType(type));
		Tag test;
		if( GlobalIDTag().isValid() ) test = GlobalIDTag();
		//else if( HaveTag("GLOBAL_ID") ) test = GetTag("GLOBAL_ID") );
		if( test.isValid() )
		{
			//bool ret = true;
			//for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype) ) if( etype & types )
			//	ret &= test.isDefined(etype);
			//return ret;
			return test.isDefined(type);
		}
		else return false;
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
193

Kirill Terekhov's avatar
Kirill Terekhov committed
194

Kirill Terekhov's avatar
Kirill Terekhov committed
195
	void Mesh::SynchronizeMarker(MarkerType marker, ElementType mask, SyncBitOp op)
Kirill Terekhov's avatar
Kirill Terekhov committed
196
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
197
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
198 199 200 201
#if defined(USE_MPI)
		if( m_state == Mesh::Parallel )
		{
			Tag t = CreateTag("TEMP_SYNC_MARKER",DATA_BULK,mask,mask,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
202
			
Kirill Terekhov's avatar
Kirill Terekhov committed
203 204 205
			//workaround for old gcc compiler
			const Element::Status SGhost = Element::Ghost;
			const Element::Status SAny = Element::Any;
Kirill Terekhov's avatar
Kirill Terekhov committed
206
			Element::Status Expr = (Element::Shared | ((op != SYNC_BIT_SET) ? SGhost : SAny));
Kirill Terekhov's avatar
Kirill Terekhov committed
207
			
Kirill Terekhov's avatar
Kirill Terekhov committed
208
			for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
209
				if( it->GetMarker(marker) && (it->GetStatus() & Expr) )
Kirill Terekhov's avatar
Kirill Terekhov committed
210
					it->Bulk(t) = 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
211
			
Kirill Terekhov's avatar
Kirill Terekhov committed
212 213 214 215
			
			
			switch(op)
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
216 217 218
				case SYNC_BIT_SET:
					ExchangeData(t,mask,0);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
219
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
220 221 222 223
						if( it->GetStatus() == Element::Ghost )
						{
							if( it->HaveData(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
224
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
225 226 227 228
					break;
				case SYNC_BIT_OR:
					ReduceData(t,mask,0,UnpackSyncMarkerOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
229
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
230 231 232 233
						if( it->GetStatus() & (Element::Ghost | Element::Shared) )
						{
							if( !it->GetMarker(marker) && it->HaveData(t) ) it->SetMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
234
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
235 236 237 238
					break;
				case SYNC_BIT_AND:
					ReduceData(t,mask,0,UnpackSyncMarkerAND);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
239
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
240 241 242 243
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
244
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
245 246 247 248
					break;
				case SYNC_BIT_XOR:
					ReduceData(t,mask,0,UnpackSyncMarkerXOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
249
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
250 251 252 253
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
254
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
255
					break;
Kirill Terekhov's avatar
Kirill Terekhov committed
256 257
			}
			
Kirill Terekhov's avatar
Kirill Terekhov committed
258
			
Kirill Terekhov's avatar
Kirill Terekhov committed
259 260
			DeleteTag(t,mask);
		}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
261 262 263 264 265
#else//USE_MPI
		(void) marker;
		(void) mask;
		(void) op;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
266
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
267
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
268
	
Kirill Terekhov's avatar
Kirill Terekhov committed
269 270
	ElementType Mesh::SynchronizeElementType(ElementType etype)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
271
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
272 273
		ElementType etypeout = etype;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
274
		REPORT_MPI(MPI_Allreduce(&etype,&etypeout,1,INMOST_MPI_DATA_BULK_TYPE,MPI_BOR,GetCommunicator()));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
275
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
276 277
		return etypeout;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
278 279
	Storage::integer Mesh::TotalNumberOf(ElementType mask)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
280
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
281 282 283 284 285
		Storage::integer number = 0, ret = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
			if( it->GetStatus() != Element::Ghost ) number++;
		ret = number;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
286
		REPORT_MPI(MPI_Allreduce(&number,&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
287
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
288
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
289 290
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
291
	Storage::integer Mesh::EnumerateSet(const ElementSet & set, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
292
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
293
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
294 295 296 297
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
298 299
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
300
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
301
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
302
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
303
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
304
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
305 306
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
307
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
308 309
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
310
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
311
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
312 313
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
314
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
315
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
316
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
317
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
318 319
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
320
	Storage::integer Mesh::Enumerate(const HandleType * set, enumerator n, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
321
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
322
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
323 324 325 326
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
327 328
		for(const HandleType * it = set; it != set+n; ++it)
			if( GetStatus(*it) != Element::Ghost && (define_sparse || HaveData(*it,num_tag))) number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
329
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
330
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
331
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
332
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
333
		for(const HandleType * it = set; it != set+n; ++it)
334
			if( GetStatus(*it) == Element::Owned && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
335
		for(const HandleType * it = set; it != set+n; ++it)
336
			if( GetStatus(*it) == Element::Shared && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
337
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
338 339
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
340
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
341
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
342
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
343
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
344 345
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
346
	Storage::integer Mesh::Enumerate(ElementType mask, Tag num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
347
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
348
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
349 350 351 352
		Storage::integer shift = 0, ret = 0;
#if defined(USE_MPI)
		Storage::integer number = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
353
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
354
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
355
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
356
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
357
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
358 359
		shift += start;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
360
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
361 362
				it->Integer(num_tag) = shift++;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
363
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
364
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
365
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
366 367
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
368
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
369
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
370
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
371
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
372 373 374 375 376
		return ret;
	}
	
	Storage::real Mesh::Integrate(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
377
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
378 379
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
380
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
381 382 383
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
384
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
385 386
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
387 388
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
389
	
Kirill Terekhov's avatar
Kirill Terekhov committed
390 391 392
	
	Storage::integer Mesh::Integrate(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
393
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
394 395
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
396
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
397 398 399
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
400
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
401 402
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
403
	
Kirill Terekhov's avatar
Kirill Terekhov committed
404 405
	void Mesh::Integrate(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
406
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
407
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
408 409
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
410
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
411
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
412
#else//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
413 414
		(void) input;
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
415
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
416
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
417
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
418
	
Kirill Terekhov's avatar
Kirill Terekhov committed
419 420
	void Mesh::Integrate(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
421
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
422
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
423 424
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
425
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
426
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
427 428
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
429
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
430
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
431
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
432
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
433 434 435
	
	Storage::integer Mesh::ExclusiveSum(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
436
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
437 438
		Storage::integer output = 0;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
439
		REPORT_MPI(MPI_Scan(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
440
		output -= input;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
441 442 443
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
444
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
445 446 447
		return output;
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
448
	Storage::real Mesh::Integrate(const Tag & t, enumerator entry, ElementType mask)
Kirill Terekhov's avatar
Kirill Terekhov committed
449
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
450
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
451 452
		Storage::real output = 0, input = 0;
		for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
453
			if( GetStatus(*it) != Element::Ghost && HaveData(*it,t) )
Kirill Terekhov's avatar
Kirill Terekhov committed
454 455 456 457
			{
				real_array arr = RealArray(*it,t);
				if( arr.size() > entry ) input += arr[entry];
			}
Kirill Terekhov's avatar
Kirill Terekhov committed
458 459
		output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
460
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
461
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
462
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
463 464 465
		return output;
	}
	
466 467
	Storage::real Mesh::AggregateMax(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
468
		ENTER_FUNC();
469 470
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
471
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
472 473 474
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
475
		EXIT_FUNC();
476 477
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
478
	
479 480
	Storage::integer Mesh::AggregateMax(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
481
		ENTER_FUNC();
482 483
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
484
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
485 486 487
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
488
		EXIT_FUNC();
489 490
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
491
	
Kirill Terekhov's avatar
Kirill Terekhov committed
492 493
	void Mesh::AggregateMax(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
494
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
495
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
496 497
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
498
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
499
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
500 501
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
502
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
503
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
504
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
505
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
506
	
Kirill Terekhov's avatar
Kirill Terekhov committed
507 508
	void Mesh::AggregateMax(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
509
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
510
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
511 512
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
513
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
514
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
515 516
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
517
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
518
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
519
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
520
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
521
	
Kirill Terekhov's avatar
Kirill Terekhov committed
522 523
	Storage::real Mesh::AggregateMin(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
524
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
525 526
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
527
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
528 529 530
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
531
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
532 533 534 535 536
		return output;
	}
	
	Storage::integer Mesh::AggregateMin(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
537
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
538 539
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
540
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
541 542 543
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
544
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
545 546
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
547
	
Kirill Terekhov's avatar
Kirill Terekhov committed
548 549
	void Mesh::AggregateMin(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
550
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
551
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
552 553
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
554
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
555
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
556 557
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
558
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
559
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
560
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
561
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
562
	
Kirill Terekhov's avatar
Kirill Terekhov committed
563 564
	void Mesh::AggregateMin(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
565
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
566
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
567 568
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
569
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
570
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
571 572
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
573
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
574
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
575
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
576
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
577 578
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
579
	INMOST_MPI_Comm Mesh::GetCommunicator() const
Kirill Terekhov's avatar
Kirill Terekhov committed
580 581 582
	{
#if defined(USE_MPI)
		return comm;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
583
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
584
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
585
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
586 587
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
588
	int Mesh::GetProcessorRank() const
Kirill Terekhov's avatar
Kirill Terekhov committed
589 590 591 592 593
	{
#if defined(USE_MPI)
		int rank;
		MPI_Comm_rank(comm,&rank);
		return rank;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
594
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
595
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
596
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
597 598
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
599
	int Mesh::GetProcessorsNumber() const
Kirill Terekhov's avatar
Kirill Terekhov committed
600 601 602 603 604
	{
#if defined(USE_MPI)
		int size;
		MPI_Comm_size(comm,&size);
		return size;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
605
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
606
		return 1;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
607
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
608 609 610 611 612 613 614 615
	}	
	
	void Mesh::Initialize(int * argc, char *** argv)
	{
#if defined(USE_MPI)
		int test;
		MPI_Initialized(&test);
        if( test == 0 ) MPI_Init(argc,argv);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
616 617 618 619
#else //USE_MPI
		(void) argc;
		(void) argv;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
620 621 622
#if defined(USE_PARALLEL_WRITE_TIME)
		atexit(Mesh::AtExit);
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
623 624 625 626 627 628 629 630
	}
	void Mesh::Finalize()
	{
#if defined(USE_MPI)
		int test = 0;
		MPI_Finalized(&test);
		if( !test )
			MPI_Finalize();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
631
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
632 633
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
634
    INMOST_MPI_Group Mesh::GetGroup() const
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
635 636 637 638 639 640 641
    {
        INMOST_MPI_Group ret = INMOST_MPI_GROUP_EMPTY;
#if defined(USE_MPI)
        MPI_Comm_group(GetCommunicator(), &ret);
#endif
        return ret;
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
642 643 644 645
	
	void Mesh::SetCommunicator(INMOST_MPI_Comm _comm)
	{
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
646 647 648 649 650 651
		tag_shared = CreateTag("PROTECTED_STATUS",DATA_BULK,CELL | FACE | EDGE | NODE,NONE,1);
		tag_owner = CreateTag("OWNER_PROCESSOR",DATA_INTEGER, CELL | FACE | EDGE | NODE,NONE,1);
		tag_processors = CreateTag("PROCESSORS_LIST",DATA_INTEGER, MESH | NODE | EDGE | FACE | CELL,NONE);
		tag_layers = CreateTag("LAYERS",DATA_INTEGER,MESH,NONE,1);
		tag_bridge = CreateTag("BRIDGE",DATA_INTEGER,MESH,NONE,1);
		tag_sendto = CreateTag("PROTECTED_SENDTO",DATA_INTEGER, CELL | FACE | EDGE | NODE, CELL | FACE | EDGE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
652 653

#if defined(USE_MPI)
654
		randomizer = Random();
Kirill Terekhov's avatar
Kirill Terekhov committed
655 656 657 658 659 660 661 662 663 664 665
		
		parallel_strategy = 1;
		parallel_file_strategy = 1;

		

		Mesh::Initialize(NULL,NULL);
		//~ MPI_Comm_dup(_comm,&comm);
		comm = _comm;
		{
			INMOST_DATA_BIG_ENUM_TYPE t = pmid;
666
			REPORT_MPI(MPI_Allreduce(&t,&parallel_mesh_unique_id,1,INMOST_MPI_DATA_BIG_ENUM_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
667 668 669 670 671
			pmid = parallel_mesh_unique_id+1;
		}
		m_state = Mesh::Parallel;


Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
672
#if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
		int err;
		REPORT_MPI(err = MPI_Alloc_mem(GetProcessorsNumber()*sizeof(unsigned)*2,MPI_INFO_NULL,&shared_space));
		if( err )
		{
			int errclass;
			MPI_Error_class(err,&errclass);
			std::cout << "Cannot allocate shared space of size " << GetProcessorsNumber()*sizeof(unsigned)*2 << " reason is "; 
			switch(err)
			{
				case MPI_SUCCESS: std::cout << "success"; break;
				case MPI_ERR_INFO: std::cout << "bad info"; break;
				case MPI_ERR_ARG: std::cout << "bad argument"; break;
				case MPI_ERR_NO_MEM: std::cout << "no memory"; break;
			}
			std::cout << std::endl;
			MPI_Abort(comm,err);
		}
		REPORT_MPI(MPI_Win_create(shared_space,sizeof(unsigned)*GetProcessorsNumber()*2,sizeof(unsigned),MPI_INFO_NULL,comm,&window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
691 692 693 694
#endif //USE_MPI_P2P
#else //USE_MPI
		(void) _comm;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
695 696 697 698 699 700
		EXIT_FUNC();
	}
	
#if defined(USE_PARALLEL_WRITE_TIME)
	void Mesh::Enter() { tab++; }
	void Mesh::Exit() {tab--; }
Kirill Terekhov's avatar
Kirill Terekhov committed
701
	std::ostream & Mesh::WriteTab(std::ostream & f)
Kirill Terekhov's avatar
Kirill Terekhov committed
702 703 704 705 706 707 708 709 710
	{
		for(int i = 0; i < tab; i++)
			f << "   ";
		return f;
	}
	std::fstream & Mesh::GetStream()
	{
		return out_time;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
711 712
	void Mesh::FinalizeFile()
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
713 714
    //std::stringstream str;
    if( tab > 1 )
Kirill Terekhov's avatar
Kirill Terekhov committed
715
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
716
      out_time << "<TEXT><![CDATA[Died!]]></TEXT>\n";// << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
717
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
718
    while(tab > 1)
Kirill Terekhov's avatar
Kirill Terekhov committed
719
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
720 721
      out_time << "<TIME>-1</TIME>\n</FUNCTION>\n";// << std::endl; 
      Exit(); 
Kirill Terekhov's avatar
Kirill Terekhov committed
722 723
		}
		out_time << "</Debug>" << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
724 725 726
    //out_time << str;
    //out_time.flush();
    out_time.close();
Kirill Terekhov's avatar
Kirill Terekhov committed
727
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
728
#endif //USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
729 730 731
	
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
732
	void determine_my_procs_low(Mesh * m, HandleType h, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
733
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
734 735 736
		Element::adj_type const & subelements = m->LowConn(h);
		Element::adj_type::const_iterator i = subelements.begin();
		Storage::integer_array p = m->IntegerArrayDV(*i,m->ProcessorsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
737 738 739 740 741
		result.clear();
		result.insert(result.end(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
742 743
			Storage::integer_array q = m->IntegerArrayDV(*i,m->ProcessorsTag());
			intersection.resize(std::max(static_cast<unsigned>(result.size()),static_cast<unsigned>(q.size())));
Kirill Terekhov's avatar
Kirill Terekhov committed
744 745 746 747 748 749 750
			dynarray<Storage::integer,64>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
751
	void determine_my_procs_high(Mesh * m, HandleType h, const Tag & procs, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
752
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
753
		Element::adj_type const & overelements = m->HighConn(h);
Kirill Terekhov's avatar
Kirill Terekhov committed
754
		if( overelements.empty() ) return;
Kirill Terekhov's avatar
Kirill Terekhov committed
755
		Element::adj_type::const_iterator i = overelements.begin();
Kirill Terekhov's avatar
Kirill Terekhov committed
756 757 758
		result.clear();
		while(i != overelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
759
			Storage::integer_array q = m->IntegerArrayDV(*i,procs);
Kirill Terekhov's avatar
Kirill Terekhov committed
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
			intersection.resize(result.size()+q.size());
			dynarray<Storage::integer,64>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
	/*
	void determine_my_procs_low(adjacent<Element> & subelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( subelements.empty() ) return;
		adjacent<Element>::iterator i = subelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size());
			std::vector<Storage::integer>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	
	void determine_my_procs_high(adjacent<Element> & overelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( overelements.empty() ) return;
		adjacent<Element>::iterator i = overelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != overelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size()+q.size());
			std::vector<Storage::integer>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	*/
	/*
	void Mesh::MarkShared(ElementType mask)
	{
		for(ElementType etype = NODE; etype <= CELL; etype = etype << 1 ) if( etype & mask )
		for(Mesh::iteratorElement it = BeginElement(etype); it != EndElement(); it++)
		{
			int owner;
			Storage::integer_array v = it->IntegerArrayDV(tag_processors);
			std::sort(v.begin(),v.end());
			if( v.empty() )
			{
				owner = mpirank;
				v.push_back(mpirank);
			}
			else
				owner = std::min(mpirank,v[0]);
			
			it->IntegerDF(tag_owner) = owner;
			
			if( mpirank == owner )
			{
				if( v.size() == 1 )
					it->BulkDF(tag_shared) = Element::Owned;
				else
					it->BulkDF(tag_shared) = Element::Shared;
			}
			else
				it->BulkDF(tag_shared) = Element::Ghost;
		}
		
		ComputeSharedProcs();
		RecomputeParallelStorage(mask);
		AssignGlobalID(mask);
		
		//have to do it for all types
#if defined(USE_PARALLEL_STORAGE)
		for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
		for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)			
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
#endif
	}
	*/
	
	
856
	__INLINE bool point_in_bbox(Storage::real * p, Storage::real bbox[6], unsigned dim, Storage::real eps)
Kirill Terekhov's avatar
Kirill Terekhov committed
857 858 859
	{
		bool ret = true;
		for(unsigned k = 0; k < dim; k++) 
860
			ret &= (p[k] > bbox[k]-eps && p[k] < bbox[dim+k]+eps);
Kirill Terekhov's avatar
Kirill Terekhov committed
861 862 863 864 865 866
		return ret;
	}
	
	__INLINE bool compare_coord(Storage::real * a, Storage::real * b, INMOST_DATA_ENUM_TYPE dim, Storage::real eps)
	{
		for(INMOST_DATA_ENUM_TYPE i = 0; i <  dim; i++)
Kirill Terekhov's avatar
Kirill Terekhov committed
867
			if( ::fabs(a[i]-b[i]) > eps ) return a[i] <= b[i];
Kirill Terekhov's avatar
Kirill Terekhov committed
868 869
		return true;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
870 871 872 873 874

	class MappingComparator
	{
	public: bool operator () (const std::pair<int,int> & a, const std::pair<int,int> & b) {return a.first < b.first;}
	};
Kirill Terekhov's avatar
Kirill Terekhov committed
875 876 877 878
	
	void Mesh::ResolveShared()
	{
		ENTER_FUNC();
879
		
Kirill Terekhov's avatar
Kirill Terekhov committed
880 881
#if defined(USE_MPI)
		if( m_state == Mesh::Serial ) SetCommunicator(INMOST_MPI_COMM_WORLD);
882
		if( tag_global_id.isValid() ) tag_global_id = DeleteTag(tag_global_id,CELL | EDGE | FACE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
883
		integer dim = GetDimensions();
Kirill Terekhov's avatar
Kirill Terekhov committed
884 885 886 887 888
		int sendsize;
		int mpirank = GetProcessorRank(),mpisize = GetProcessorsNumber();
#if defined(USE_PARALLEL_STORAGE)
		shared_elements.clear();
		ghost_elements.clear();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
889
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
890 891
		//determine which bboxes i intersect
		dynarray<int,64> procs;
Kirill Terekhov's avatar
Kirill Terekhov committed
892 893 894 895 896
		Storage::real bbox[6]; //local bounding box
		std::vector<Storage::real> bboxs(mpisize*6);
		//Compute local bounding box containing nodes.
		//Will be more convinient to compute (or store)
		//and communicate local octree over all the nodes.
Kirill Terekhov's avatar
Kirill Terekhov committed
897
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
898 899 900 901
		{
			bbox[k] = 1e20;
			bbox[k+dim] = -1e20;
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
902 903 904
#if defined(USE_OMP)
#pragma omp parallel
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
905
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
906
			real bbox0[6];
Kirill Terekhov's avatar
Kirill Terekhov committed
907
			for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
908
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
				bbox0[k] = 1e20;
				bbox0[k+dim] = -1e20;
			}
#if defined(USE_OMP)
#pragma omp for
#endif
			for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
			{
				Node it = NodeByLocalID(nit);
				Storage::real_array arr = it->Coords();
				for(integer k = 0; k < dim; k++)
				{
					if( arr[k] < bbox0[k] ) bbox0[k] = arr[k];
					if( arr[k] > bbox0[k+dim] ) bbox0[k+dim] = arr[k];
				}
			}
#if defined(USE_OMP)
#pragma omp critical
#endif
			{
				for(integer k = 0; k < dim; k++)
				{
					if( bbox0[k] < bbox[k] ) bbox[k] = bbox0[k];
Kirill Terekhov's avatar
Kirill Terekhov committed
932
					if( bbox0[k+dim] > bbox[k+dim] ) bbox[k+dim] = bbox0[k+dim];
Kirill Terekhov's avatar
Kirill Terekhov committed
933
				}
Kirill Terekhov's avatar
Kirill Terekhov committed
934 935
			}
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
936
		REPORT_VAL("dim",dim);
937
		assert(dim*2 <= 6);
Kirill Terekhov's avatar
Kirill Terekhov committed
938
		// write down bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
939
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
940 941 942 943
		{
			REPORT_VAL("min",bbox[k]);
			REPORT_VAL("max",bbox[dim+k]);
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
944
		// communicate bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
945
		REPORT_MPI(MPI_Allgather(&bbox[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,&bboxs[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
946
		// find all processors that i communicate with
Kirill Terekhov's avatar
Kirill Terekhov committed
947 948 949 950
		for(int k = 0; k < mpisize; k++)
			if( k != mpirank )
			{
				bool flag = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
951
				for(integer q = 0; q < dim; q++)
952
					flag &= !((bbox[q]-GetEpsilon() > bboxs[k*dim*2+q+dim]) || (bbox[dim+q]+GetEpsilon() < bboxs[k*dim*2+q]));
Kirill Terekhov's avatar
Kirill Terekhov committed
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
				if( flag ) procs.push_back(k);
			}
		REPORT_VAL("neighbour processors",procs.size());
		
		//~ if( procs.empty() )
		//~ {
			//~ REPORT_STR("no processors around - all elements are owned");
			//~ for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
			//~ {
				//~ it->IntegerArrayDV(tag_processors).resize(1);
				//~ it->IntegerDF(tag_owner) = it->IntegerDV(tag_processors) = mpirank;
				//~ it->BulkDF(tag_shared) = Element::Owned;
			//~ }
			//~ ComputeSharedProcs();
			//~ RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
			//~ AssignGlobalID(CELL | EDGE | FACE | NODE);
		//~ }
		//~ else
		{
			bool same_boxes = true, same_box;
			for(int k = 0; k < mpisize && same_boxes; k++)
			{
				same_box = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
976
				for(integer j = 0; j < dim*2; j++)
977
					same_box &= ::fabs(bbox[j] - bboxs[k*dim*2+j]) < GetEpsilon();
Kirill Terekhov's avatar
Kirill Terekhov committed
978 979 980 981 982 983
				same_boxes &= same_box;
			}
			
			if( same_boxes )
			{
				REPORT_STR("All bounding boxes are the same - assuming that mesh is replicated over all nodes");
Kirill Terekhov's avatar
Kirill Terekhov committed
984 985
				//for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
				for(ElementType etype = NODE; etype <= CELL; etype = NextElementType(etype) )
Kirill Terekhov's avatar
Kirill Terekhov committed
986
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
#if defined(USE_OMP)
#pragma omp parallel for
#endif
					for(integer eit = 0; eit < LastLocalID(etype); ++eit) if( isValidElement(etype,eit) )
					{
						Element it = ElementByLocalID(etype,eit);
						integer_array arr = it->IntegerArrayDV(tag_processors);
						arr.resize(mpisize);
						for(int k = 0; k < mpisize; k++)
							arr[k] = k;
						it->IntegerDF(tag_owner) = 0;
						if( mpirank == 0 )
							SetStatus(it->GetHandle(),Element::Shared);
						else
							SetStatus(it->GetHandle(),Element::Ghost);
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1003 1004 1005 1006 1007 1008 1009 1010 1011
				}
				ComputeSharedProcs();
				RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
				AssignGlobalID(CELL | EDGE | FACE | NODE);
#if defined(USE_PARALLEL_STORAGE)
				for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1012 1013
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1014
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1015
				for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1016 1017 1018
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1019 1020
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1021
					}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1022
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
1023 1024 1025 1026 1027 1028
			}
			else
			{
				double time = Timer();
				Storage::real epsilon = GetEpsilon();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1029
				GeomParam table;
1030
				for(ElementType etype = EDGE; etype <= CELL; etype = NextElementType(etype))
Kirill Terekhov's avatar
Kirill Terekhov committed
1031 1032 1033 1034 1035 1036 1037 1038 1039
					if( !HaveGeometricData(CENTROID,etype) )
						table[CENTROID] |= etype;
				PrepareGeometricData(table);
				
				time = Timer() - time;
			
				REPORT_STR("Prepare geometric data");
				REPORT_VAL("time",time);
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1040 1041 1042 1043 1044
				//for(iteratorNode it = BeginNode(); it != EndNode(); it++)
#if defined(USE_OMP)
#pragma omp parallel for
#endif
				for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1045
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1046
					Node it = NodeByLocalID(nit);
Kirill Terekhov's avatar
Kirill Terekhov committed
1047 1048 1049 1050 1051 1052
					Storage::integer_array arr = it->IntegerArrayDV(tag_processors);
					arr.resize(1);
					arr[0] = mpirank;
				}
			

Kirill Terekhov's avatar
Kirill Terekhov committed
1053
				buffer_type exch_data;
Kirill Terekhov's avatar
Kirill Terekhov committed
1054 1055
				std::vector< INMOST_DATA_REAL_TYPE > unpack_real;
				std::vector< INMOST_DATA_REAL_TYPE > pack_real;
Kirill Terekhov's avatar
Kirill Terekhov committed
1056
				element_set sorted_nodes;
Kirill Terekhov's avatar
Kirill Terekhov committed
1057 1058 1059 1060 1061 1062 1063 1064 1065
				
				sorted_nodes.reserve(NumberOfNodes());
				
				
				time = Timer();
				
				
				for(iteratorNode n = BeginNode(); n != EndNode(); n++)
				{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1066 1067
					real_array c = n->Coords();
					for(real_array::size_type k = 0; k < procs.size(); k++)
1068
						if( point_in_bbox(c.data(),bboxs.data()+procs[k]*dim*2,dim,GetEpsilon()) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1069
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1070
							sorted_nodes.push_back(*n);
Kirill Terekhov's avatar
Kirill Terekhov committed
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
							break;
						}
				}
				
				time = Timer() - time;
				REPORT_STR("Prepare array of nodes");
				REPORT_VAL("time",time);
				REPORT_VAL("share nodes", sorted_nodes.size());
				REPORT_VAL("total nodes", NumberOfNodes());
				
				
				
				time = Timer();
				if( !sorted_nodes.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1085 1086
					std::sort(sorted_nodes.begin(),sorted_nodes.end(),CentroidComparator(this));
					//qsort(&sorted_nodes[0],sorted_nodes.size(),sizeof(Element *),CompareElementsCCentroid);
Kirill Terekhov's avatar
Kirill Terekhov committed
1087 1088 1089 1090 1091 1092 1093 1094 1095
				time = Timer() - time;
				REPORT_STR("Sort nodes");
				REPORT_VAL("time",time);
				
				
				pack_real.reserve(sorted_nodes.size()*dim);
				
				
				time = Timer();
Kirill Terekhov's avatar
Kirill Terekhov committed
1096
				for(element_set::iterator it = sorted_nodes.begin(); it != sorted_nodes.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1097
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1098
					Storage::real_array arr = RealArrayDF(*it,CoordsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
1099 1100 1101 1102 1103 1104 1105 1106 1107
					pack_real.insert(pack_real.end(),arr.begin(),arr.end());
				}
				time = Timer() - time;
				REPORT_STR("Gather coordinates");
				REPORT_VAL("time",time);
				
				
				time = Timer();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1108
				MPI_Pack_size(static_cast<int>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm,&sendsize);
Kirill Terekhov's avatar
Kirill Terekhov committed
1109 1110
				exch_data.resize(sendsize);
				int position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1111
				if( sendsize > 0 ) MPI_Pack(&pack_real[0],static_cast<INMOST_MPI_SIZE>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,&exch_data[0],static_cast<INMOST_MPI_SIZE>(exch_data.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1112 1113 1114 1115 1116 1117 1118
				
				
				time = Timer() - time;
				REPORT_STR("Pack coordinates");
				REPORT_VAL("time",time);

				
1119
				
Kirill Terekhov's avatar
Kirill Terekhov committed
1120 1121 1122 1123
				{
					std::vector< MPI_Request > send_reqs, recv_reqs;
					exch_buffer_type send_buffs(procs.size()), recv_buffs(procs.size());
					std::vector<int> done;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1124
//~ #if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
1125 1126
					//~ unsigned * sendsizeall = shared_space;
					//~ unsigned usend[2] = {sendsize,pack_real.size()};
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1127
					//~ REPORT_MPI(MPI_Win_fence(0,window)); //start exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1128 1129
					//~ for(unsigned k = 0; k < procs.size(); k++)
						//~ REPORT_MPI(MPI_Put(usend,2,MPI_UNSIGNED,procs[k],mpirank*2,2,MPI_UNSIGNED,window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1130
					//~ REPORT_MPI(MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED,window)); //end exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1131 1132 1133
//~ #else
					std::vector<unsigned> sendsizeall(mpisize*2);
					int pack_size2 = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1134
					unsigned usend[2] = {static_cast<unsigned>(sendsize),static_cast<unsigned>(pack_real.size())};
Kirill Terekhov's avatar
Kirill Terekhov committed
1135
					MPI_Pack_size(2,MPI_UNSIGNED,comm,&pack_size2);
Kirill Terekhov's avatar
Kirill Terekhov committed
1136
					for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1137 1138 1139 1140
					{
						send_buffs[k].first = procs[k];
						send_buffs[k].second.resize(pack_size2);
						position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1141
						MPI_Pack(usend,2,MPI_UNSIGNED,&send_buffs[k].second[0],static_cast<INMOST_MPI_SIZE>(send_buffs[k].second.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1142 1143 1144 1145 1146 1147 1148 1149 1150
						recv_buffs[k].first = procs[k];
						recv_buffs[k].second.resize(pack_size2);
					}
					ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
					while( !(done = FinishRequests(recv_reqs)).empty() )
					{
						for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
						{
							position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1151
							MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&sendsizeall[procs[*qt]*2],2,MPI_UNSIGNED,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1152 1153 1154
						}
					}
					if( !send_reqs.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1155
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
1156
						REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
Kirill Terekhov's avatar
Kirill Terekhov committed
1157
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1158 1159 1160 1161 1162 1163
					//~ REPORT_MPI(MPI_Allgather(usend,2,MPI_UNSIGNED,&sendsizeall[0],2,MPI_UNSIGNED,comm));
//~ #endif
					double time2 = Timer();
					{
						
						
Kirill Terekhov's avatar
Kirill Terekhov committed
1164
						for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
						{
							send_buffs[k].first = procs[k];
							send_buffs[k].second = exch_data;
							recv_buffs[k].first = procs[k];
							recv_buffs[k].second.resize(sendsizeall[procs[k]*2]);
						}
						
						//PrepareReceiveInner(send_buffs,recv_buffs);
						ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
						
						
						
						while( !(done = FinishRequests(recv_reqs)).empty() )
						{
							for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
							{
								time = Timer();
								REPORT_STR("receive node coordinates");
								REPORT_VAL("processor",recv_buffs[*qt].first);
								int count = 0;
								int position = 0;
								unpack_real.resize(sendsizeall[recv_buffs[*qt].first*2+1]);
Kirill Terekhov's avatar
Kirill Terekhov committed
1187
								MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&unpack_real[0],static_cast<INMOST_MPI_SIZE>(unpack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1188 1189 1190 1191
								std::vector<Storage::real>::iterator it1 = pack_real.begin() , it2 = unpack_real.begin();
								while(it1 != pack_real.end() && it2 != unpack_real.end() )
								{
									int res = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1192
									for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1193
										if( ::fabs((*(it1+k))-(*(it2+k))) > epsilon )
Kirill Terekhov's avatar
Kirill Terekhov committed
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
										{
											if( (*(it1+k)) < (*(it2+k)) ) res = -1;
											else res = 1;
											break;
										}
									if( res < 0 ) 
										it1 += dim;
									else if( res > 0 ) 
										it2 += dim;
									else
									{
										count++;
Kirill Terekhov's avatar
Kirill Terekhov committed
1206
										IntegerArrayDV(sorted_nodes[(it1-pack_real.begin())/dim],tag_processors).push_back(recv_buffs[*qt].first);
Kirill Terekhov's avatar
Kirill Terekhov committed
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
										it1 += dim;
										it2 += dim;
									}
								}
								REPORT_VAL("intersected coords",count);
								time = Timer() - time;
								REPORT_STR("Intersect coordinates");
								REPORT_VAL("time",time);
							}
						}
						if( !send_reqs.empty() )