parallel.cpp 208 KB
Newer Older
Kirill Terekhov's avatar
Kirill Terekhov committed
1
#ifdef _MSC_VER //kill some warnings
2
#ifndef _SCL_SECURE_NO_WARNINGS
Kirill Terekhov's avatar
Kirill Terekhov committed
3 4
#define _SCL_SECURE_NO_WARNINGS
#endif
5
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
6

Kirill Terekhov's avatar
Kirill Terekhov committed
7 8 9 10 11 12 13
#include "inmost.h"


#if defined(USE_MESH)
#include <iostream>
#include <fstream>
#include <sstream>
14
#include <string>
Kirill Terekhov's avatar
Kirill Terekhov committed
15

16
using namespace std;
Kirill Terekhov's avatar
Kirill Terekhov committed
17 18 19 20 21 22 23 24 25

#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
#endif

#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
26 27 28
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
29 30
#else
#define REPORT_MPI(x) x
Kirill Terekhov's avatar
Kirill Terekhov committed
31 32 33 34 35
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE()  {}
Kirill Terekhov's avatar
Kirill Terekhov committed
36 37
#endif

38 39 40 41 42 43 44 45

#if defined(__LINUX__) || defined(__linux__) || defined(__APPLE__)
#include <unistd.h>
#define PROCESSID getpid()
#else
#define PROCESSID -1
#endif

Kirill Terekhov's avatar
Kirill Terekhov committed
46 47
namespace INMOST
{
48
    static int block_recursion = 0;
49 50 51 52

    std::string ro()
    {
        int rank = 0;
53
#ifdef USE_MPI
54
        MPI_Comm_rank(MPI_COMM_WORLD,&rank);
55
#endif
56 57 58 59 60 61
        std::stringstream ss;
        for (int i = 0; i < rank; i++)
            ss << "   ";
        return ss.str();
    }
    
Kirill Terekhov's avatar
Kirill Terekhov committed
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
	//////////////////////////////
	/// REDUCTION FUNCTIONS    ///
	//////////////////////////////
	void DefaultUnpack(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size == 0 )
		{
			if( tag.isDefined(element->GetElementType()) ) 
			{
				if( tag.isSparse(element->GetElementType()) )
				{
					if( element->HaveData(tag) )
						element->DelData(tag); 
				}
				else if( tag.GetSize() == ENUMUNDEF )
					element->SetDataSize(tag,size);
				else
				{
					std::cerr << "received zero size for dense nonzero tag" << std::endl;
					assert(false);
				}
			}
			return;
		}
		if( !element->HaveData(tag) )
			element->SetDataSize(tag,size);
		else if( size != element->GetDataSize(tag) )
		{
			if( tag.GetSize() == ENUMUNDEF )
				element->SetDataSize(tag,size);
			else
			{
				assert(false);
			}
		}
		element->SetData(tag,0,size,data);
	}


	void UnpackOnSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
105
      const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data));
Kirill Terekhov's avatar
Kirill Terekhov committed
106 107 108 109 110 111 112 113 114 115
			Storage::integer_array arr = e->IntegerArray(tag);
			arr.push_back(recv[0]);
		}
	}


	void UnpackSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
      for(INMOST_DATA_ENUM_TYPE k = 0; k < size; k+=2 )
      {
			  bool flag = true;
			  const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data))+k;
			  Storage::integer_array arr = e->IntegerArray(tag);
      
        for(Storage::integer_array::iterator it = arr.begin(); it != arr.end(); it+=2)
        {
          if( *it == recv[0] )
				  {
					  flag = false;
					  break;
				  }
        }
			  if( flag ) 
			  {
				  arr.push_back(recv[0]);
				  arr.push_back(recv[1]);
			  }
      }
Kirill Terekhov's avatar
Kirill Terekhov committed
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
		}
	}
	


	void DeleteUnpack(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
	void RedistUnpack(const Tag & tag,const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			Storage::integer_array p1 = e->IntegerArray(tag);
			const Storage::integer * p2 = static_cast<const Storage::integer *>(static_cast<const void *>(data));
			dynarray<Storage::integer,64> result(p1.size()+size);
			dynarray<Storage::integer,64>::iterator end;
			end = std::set_union(p1.begin(),p1.end(),p2,p2+size,result.begin());
			result.resize(end-result.begin());
			p1.clear();
			p1.insert(p1.end(),result.begin(),result.end());
		}
	}

	void UnpackLayersMarker(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
180

Kirill Terekhov's avatar
Kirill Terekhov committed
181
	void UnpackSyncMarkerOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
182
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
183
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
184 185 186
		element->Bulk(tag) |= *data;
	}

Kirill Terekhov's avatar
Kirill Terekhov committed
187
	void UnpackSyncMarkerXOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
188
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
189
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
190 191 192 193
		element->Bulk(tag) ^= *data;
	}

	
Kirill Terekhov's avatar
Kirill Terekhov committed
194
	void UnpackSyncMarkerAND(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
195
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
196
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
197 198 199
		element->Bulk(tag) &= *data;
	}

Kirill Terekhov's avatar
Kirill Terekhov committed
200 201 202
	//////////////////////////////
	/// REDUCTION FUNCTIONS END///
	//////////////////////////////
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
	bool Mesh::HaveGlobalID(ElementType type) const
	{
		assert(OneType(type));
		Tag test;
		if( GlobalIDTag().isValid() ) test = GlobalIDTag();
		//else if( HaveTag("GLOBAL_ID") ) test = GetTag("GLOBAL_ID") );
		if( test.isValid() )
		{
			//bool ret = true;
			//for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype) ) if( etype & types )
			//	ret &= test.isDefined(etype);
			//return ret;
			return test.isDefined(type);
		}
		else return false;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
219

Kirill Terekhov's avatar
Kirill Terekhov committed
220

Kirill Terekhov's avatar
Kirill Terekhov committed
221
	void Mesh::SynchronizeMarker(MarkerType marker, ElementType mask, SyncBitOp op)
Kirill Terekhov's avatar
Kirill Terekhov committed
222
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
223
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
224 225 226 227
#if defined(USE_MPI)
		if( m_state == Mesh::Parallel )
		{
			Tag t = CreateTag("TEMP_SYNC_MARKER",DATA_BULK,mask,mask,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
228
			
Kirill Terekhov's avatar
Kirill Terekhov committed
229 230 231
			//workaround for old gcc compiler
			const Element::Status SGhost = Element::Ghost;
			const Element::Status SAny = Element::Any;
Kirill Terekhov's avatar
Kirill Terekhov committed
232
			Element::Status Expr = (Element::Shared | ((op != SYNC_BIT_SET) ? SGhost : SAny));
Kirill Terekhov's avatar
Kirill Terekhov committed
233
			
Kirill Terekhov's avatar
Kirill Terekhov committed
234
			for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
235
				if( it->GetMarker(marker) && (it->GetStatus() & Expr) )
Kirill Terekhov's avatar
Kirill Terekhov committed
236
					it->Bulk(t) = 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
237
			
Kirill Terekhov's avatar
Kirill Terekhov committed
238 239 240 241
			
			
			switch(op)
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
242 243 244
				case SYNC_BIT_SET:
					ExchangeData(t,mask,0);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
245
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
246 247 248 249
						if( it->GetStatus() == Element::Ghost )
						{
							if( it->HaveData(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
250
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
251 252 253 254
					break;
				case SYNC_BIT_OR:
					ReduceData(t,mask,0,UnpackSyncMarkerOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
255
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
256 257 258 259
						if( it->GetStatus() & (Element::Ghost | Element::Shared) )
						{
							if( !it->GetMarker(marker) && it->HaveData(t) ) it->SetMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
260
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
261 262 263 264
					break;
				case SYNC_BIT_AND:
					ReduceData(t,mask,0,UnpackSyncMarkerAND);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
265
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
266 267 268 269
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
270
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
271 272 273 274
					break;
				case SYNC_BIT_XOR:
					ReduceData(t,mask,0,UnpackSyncMarkerXOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
275
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
276 277 278 279
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
280
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
281
					break;
Kirill Terekhov's avatar
Kirill Terekhov committed
282 283
			}
			
Kirill Terekhov's avatar
Kirill Terekhov committed
284
			
Kirill Terekhov's avatar
Kirill Terekhov committed
285 286
			DeleteTag(t,mask);
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
287 288 289 290 291
#else//USE_MPI
		(void) marker;
		(void) mask;
		(void) op;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
292
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
293
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
294
	
Kirill Terekhov's avatar
Kirill Terekhov committed
295 296
	ElementType Mesh::SynchronizeElementType(ElementType etype)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
297
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
298 299
		ElementType etypeout = etype;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
300
		REPORT_MPI(MPI_Allreduce(&etype,&etypeout,1,INMOST_MPI_DATA_BULK_TYPE,MPI_BOR,GetCommunicator()));
Kirill Terekhov's avatar
Kirill Terekhov committed
301
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
302 303
		return etypeout;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
304 305
	Storage::integer Mesh::TotalNumberOf(ElementType mask)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
306
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
307 308 309 310 311
		Storage::integer number = 0, ret = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
			if( it->GetStatus() != Element::Ghost ) number++;
		ret = number;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
312
		REPORT_MPI(MPI_Allreduce(&number,&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
313
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
314
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
315 316
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
317
	Storage::integer Mesh::EnumerateSet(const ElementSet & set, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
318
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
319
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
320 321 322 323
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
324 325
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
326
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
327
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
328
		shift -= number;
Kirill Terekhov's avatar
Kirill Terekhov committed
329
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
330
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
331 332
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
333
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
334 335
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
336
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
337
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
338 339
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
340
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
341
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
342
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
343
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
344 345
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
346
	Storage::integer Mesh::Enumerate(const HandleType * set, enumerator n, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
347
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
348
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
349 350 351 352
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
353 354
		for(const HandleType * it = set; it != set+n; ++it)
			if( GetStatus(*it) != Element::Ghost && (define_sparse || HaveData(*it,num_tag))) number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
355
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
356
		shift -= number;
Kirill Terekhov's avatar
Kirill Terekhov committed
357
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
358
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
359
		for(const HandleType * it = set; it != set+n; ++it)
360
			if( GetStatus(*it) == Element::Owned && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
361
		for(const HandleType * it = set; it != set+n; ++it)
362
			if( GetStatus(*it) == Element::Shared && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
363
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
364 365
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
366
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
367
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
368
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
369
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
370 371
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
372
	Storage::integer Mesh::Enumerate(ElementType mask, Tag num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
373
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
374
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
375 376 377 378
		Storage::integer shift = 0, ret = 0;
#if defined(USE_MPI)
		Storage::integer number = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
379
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
380
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
381
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
382
		shift -= number;
Kirill Terekhov's avatar
Kirill Terekhov committed
383
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
384 385
		shift += start;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
386
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
387 388
				it->Integer(num_tag) = shift++;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
389
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
390
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
391
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
392 393
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
394
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
395
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
396
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
397
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
398 399 400 401 402
		return ret;
	}
	
	Storage::real Mesh::Integrate(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
403
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
404 405
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
406
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
407 408 409
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
410
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
411 412
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
413 414
	
	
415
	
Kirill Terekhov's avatar
Kirill Terekhov committed
416 417 418
	
	Storage::integer Mesh::Integrate(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
419
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
420 421
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
422
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
423 424 425
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
426
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
427 428
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
429
	
430 431
	void Mesh::Integrate(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
432
		ENTER_FUNC();
433
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
434 435
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
436
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
437
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
438
#else//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
439 440
		(void) input;
		(void) size;
441
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
442
		EXIT_FUNC();
443
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
444
	
445 446
	void Mesh::Integrate(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
447
		ENTER_FUNC();
448
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
449 450
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
451
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
452
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
453 454
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
455
		(void) size;
456
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
457
		EXIT_FUNC();
458
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
459 460 461
	
	Storage::integer Mesh::ExclusiveSum(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
462
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
463 464
		Storage::integer output = 0;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
465
		REPORT_MPI(MPI_Scan(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
466
		output -= input;
Kirill Terekhov's avatar
Kirill Terekhov committed
467 468 469
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
470
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
471 472 473
		return output;
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
474
	Storage::real Mesh::Integrate(const Tag & t, enumerator entry, ElementType mask)
Kirill Terekhov's avatar
Kirill Terekhov committed
475
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
476
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
477 478
		Storage::real output = 0, input = 0;
		for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
479
			if( GetStatus(*it) != Element::Ghost && HaveData(*it,t) )
Kirill Terekhov's avatar
Kirill Terekhov committed
480 481 482 483
			{
				real_array arr = RealArray(*it,t);
				if( arr.size() > entry ) input += arr[entry];
			}
Kirill Terekhov's avatar
Kirill Terekhov committed
484 485
		output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
486
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
487
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
488
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
489 490 491
		return output;
	}
	
492 493
	Storage::real Mesh::AggregateMax(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
494
		ENTER_FUNC();
495 496
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
497
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
498 499 500
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
501
		EXIT_FUNC();
502 503
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
504
	
505 506
	Storage::integer Mesh::AggregateMax(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
507
		ENTER_FUNC();
508 509
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
510
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
511 512 513
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
514
		EXIT_FUNC();
515 516
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
517
	
518 519
	void Mesh::AggregateMax(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
520
		ENTER_FUNC();
521
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
522 523
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
524
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
525
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
526 527
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
528
		(void) size;
529
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
530
		EXIT_FUNC();
531
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
532
	
533 534
	void Mesh::AggregateMax(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
535
		ENTER_FUNC();
536
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
537 538
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
539
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
540
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
541 542
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
543
		(void) size;
544
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
545
		EXIT_FUNC();
546
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
547
	
548 549
	Storage::real Mesh::AggregateMin(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
550
		ENTER_FUNC();
551 552
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
553
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
554 555 556
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
557
		EXIT_FUNC();
558 559 560 561 562
		return output;
	}
	
	Storage::integer Mesh::AggregateMin(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
563
		ENTER_FUNC();
564 565
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
566
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
567 568 569
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
570
		EXIT_FUNC();
571 572
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
573
	
574 575
	void Mesh::AggregateMin(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
576
		ENTER_FUNC();
577
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
578 579
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
580
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
581
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
582 583
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
584
		(void) size;
585
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
586
		EXIT_FUNC();
587
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
588
	
589 590
	void Mesh::AggregateMin(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
591
		ENTER_FUNC();
592
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
593 594
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
595
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
596
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
597 598
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
599
		(void) size;
600
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
601
		EXIT_FUNC();
602
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
603 604
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
605
	INMOST_MPI_Comm Mesh::GetCommunicator() const
Kirill Terekhov's avatar
Kirill Terekhov committed
606 607 608
	{
#if defined(USE_MPI)
		return comm;
Kirill Terekhov's avatar
Kirill Terekhov committed
609
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
610
		return 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
611
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
612 613
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
614
	int Mesh::GetProcessorRank() const
Kirill Terekhov's avatar
Kirill Terekhov committed
615 616 617 618 619
	{
#if defined(USE_MPI)
		int rank;
		MPI_Comm_rank(comm,&rank);
		return rank;
Kirill Terekhov's avatar
Kirill Terekhov committed
620
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
621
		return 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
622
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
623 624
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
625
	int Mesh::GetProcessorsNumber() const
Kirill Terekhov's avatar
Kirill Terekhov committed
626 627 628 629 630
	{
#if defined(USE_MPI)
		int size;
		MPI_Comm_size(comm,&size);
		return size;
Kirill Terekhov's avatar
Kirill Terekhov committed
631
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
632
		return 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
633
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
634 635 636 637 638 639 640 641
	}	
	
	void Mesh::Initialize(int * argc, char *** argv)
	{
#if defined(USE_MPI)
		int test;
		MPI_Initialized(&test);
        if( test == 0 ) MPI_Init(argc,argv);
Kirill Terekhov's avatar
Kirill Terekhov committed
642 643 644 645
#else //USE_MPI
		(void) argc;
		(void) argv;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
646 647 648
#if defined(USE_PARALLEL_WRITE_TIME)
		atexit(Mesh::AtExit);
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
649 650 651 652 653 654 655 656
	}
	void Mesh::Finalize()
	{
#if defined(USE_MPI)
		int test = 0;
		MPI_Finalized(&test);
		if( !test )
			MPI_Finalize();
Kirill Terekhov's avatar
Kirill Terekhov committed
657
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
658 659
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
660
    INMOST_MPI_Group Mesh::GetGroup() const
Kirill Terekhov's avatar
Kirill Terekhov committed
661 662 663 664 665 666 667
    {
        INMOST_MPI_Group ret = INMOST_MPI_GROUP_EMPTY;
#if defined(USE_MPI)
        MPI_Comm_group(GetCommunicator(), &ret);
#endif
        return ret;
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
668 669 670 671
	
	void Mesh::SetCommunicator(INMOST_MPI_Comm _comm)
	{
		ENTER_FUNC();
672 673 674
		tag_shared = CreateTag("PROTECTED_STATUS",DATA_BULK,  ESET |CELL | FACE | EDGE | NODE,NONE,1);
		tag_owner = CreateTag("OWNER_PROCESSOR",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE,NONE,1);
		tag_processors = CreateTag("PROCESSORS_LIST",DATA_INTEGER, ESET|  MESH | NODE | EDGE | FACE | CELL,NONE);
Kirill Terekhov's avatar
Kirill Terekhov committed
675 676
		tag_layers = CreateTag("LAYERS",DATA_INTEGER,MESH,NONE,1);
		tag_bridge = CreateTag("BRIDGE",DATA_INTEGER,MESH,NONE,1);
677 678
		tag_sendto = CreateTag("PROTECTED_SENDTO",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE, ESET | CELL | FACE | EDGE | NODE);
		tag_an_id = CreateTag("PROTECTED_ID",DATA_INTEGER, CELL | FACE | EDGE | NODE, CELL | FACE | EDGE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
679 680

#if defined(USE_MPI)
681
		randomizer = Random();
Kirill Terekhov's avatar
Kirill Terekhov committed
682 683 684 685 686 687 688 689 690 691 692
		
		parallel_strategy = 1;
		parallel_file_strategy = 1;

		

		Mesh::Initialize(NULL,NULL);
		//~ MPI_Comm_dup(_comm,&comm);
		comm = _comm;
		{
			INMOST_DATA_BIG_ENUM_TYPE t = pmid;
693
			REPORT_MPI(MPI_Allreduce(&t,&parallel_mesh_unique_id,1,INMOST_MPI_DATA_BIG_ENUM_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
694 695 696 697 698
			pmid = parallel_mesh_unique_id+1;
		}
		m_state = Mesh::Parallel;


Kirill Terekhov's avatar
Kirill Terekhov committed
699
#if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
		int err;
		REPORT_MPI(err = MPI_Alloc_mem(GetProcessorsNumber()*sizeof(unsigned)*2,MPI_INFO_NULL,&shared_space));
		if( err )
		{
			int errclass;
			MPI_Error_class(err,&errclass);
			std::cout << "Cannot allocate shared space of size " << GetProcessorsNumber()*sizeof(unsigned)*2 << " reason is "; 
			switch(err)
			{
				case MPI_SUCCESS: std::cout << "success"; break;
				case MPI_ERR_INFO: std::cout << "bad info"; break;
				case MPI_ERR_ARG: std::cout << "bad argument"; break;
				case MPI_ERR_NO_MEM: std::cout << "no memory"; break;
			}
			std::cout << std::endl;
			MPI_Abort(comm,err);
		}
		REPORT_MPI(MPI_Win_create(shared_space,sizeof(unsigned)*GetProcessorsNumber()*2,sizeof(unsigned),MPI_INFO_NULL,comm,&window));
Kirill Terekhov's avatar
Kirill Terekhov committed
718 719 720 721
#endif //USE_MPI_P2P
#else //USE_MPI
		(void) _comm;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
722 723 724 725 726 727
		EXIT_FUNC();
	}
	
#if defined(USE_PARALLEL_WRITE_TIME)
	void Mesh::Enter() { tab++; }
	void Mesh::Exit() {tab--; }
Kirill Terekhov's avatar
Kirill Terekhov committed
728
	std::ostream & Mesh::WriteTab(std::ostream & f)
Kirill Terekhov's avatar
Kirill Terekhov committed
729 730 731 732 733 734 735 736 737
	{
		for(int i = 0; i < tab; i++)
			f << "   ";
		return f;
	}
	std::fstream & Mesh::GetStream()
	{
		return out_time;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
738 739
	void Mesh::FinalizeFile()
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
740 741
    //std::stringstream str;
    if( tab > 1 )
Kirill Terekhov's avatar
Kirill Terekhov committed
742
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
743
      out_time << "<TEXT><![CDATA[Died!]]></TEXT>\n";// << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
744
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
745
    while(tab > 1)
Kirill Terekhov's avatar
Kirill Terekhov committed
746
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
747 748
      out_time << "<TIME>-1</TIME>\n</FUNCTION>\n";// << std::endl; 
      Exit(); 
Kirill Terekhov's avatar
Kirill Terekhov committed
749 750
		}
		out_time << "</Debug>" << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
751 752 753
    //out_time << str;
    //out_time.flush();
    out_time.close();
Kirill Terekhov's avatar
Kirill Terekhov committed
754
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
755
#endif //USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
756 757 758
	
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
759
	void determine_my_procs_low(Mesh * m, HandleType h, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
760
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
761 762 763
		Element::adj_type const & subelements = m->LowConn(h);
		Element::adj_type::const_iterator i = subelements.begin();
		Storage::integer_array p = m->IntegerArrayDV(*i,m->ProcessorsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
764 765 766 767 768
		result.clear();
		result.insert(result.end(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
769 770
			Storage::integer_array q = m->IntegerArrayDV(*i,m->ProcessorsTag());
			intersection.resize(std::max(static_cast<unsigned>(result.size()),static_cast<unsigned>(q.size())));
Kirill Terekhov's avatar
Kirill Terekhov committed
771 772 773 774 775 776 777
			dynarray<Storage::integer,64>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
778
	void determine_my_procs_high(Mesh * m, HandleType h, const Tag & procs, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
779
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
780
		Element::adj_type const & overelements = m->HighConn(h);
Kirill Terekhov's avatar
Kirill Terekhov committed
781
		if( overelements.empty() ) return;
Kirill Terekhov's avatar
Kirill Terekhov committed
782
		Element::adj_type::const_iterator i = overelements.begin();
Kirill Terekhov's avatar
Kirill Terekhov committed
783 784 785
		result.clear();
		while(i != overelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
786
			Storage::integer_array q = m->IntegerArrayDV(*i,procs);
Kirill Terekhov's avatar
Kirill Terekhov committed
787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
			intersection.resize(result.size()+q.size());
			dynarray<Storage::integer,64>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
	/*
	void determine_my_procs_low(adjacent<Element> & subelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( subelements.empty() ) return;
		adjacent<Element>::iterator i = subelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size());
			std::vector<Storage::integer>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	
	void determine_my_procs_high(adjacent<Element> & overelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( overelements.empty() ) return;
		adjacent<Element>::iterator i = overelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != overelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size()+q.size());
			std::vector<Storage::integer>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	*/
	/*
	void Mesh::MarkShared(ElementType mask)
	{
		for(ElementType etype = NODE; etype <= CELL; etype = etype << 1 ) if( etype & mask )
		for(Mesh::iteratorElement it = BeginElement(etype); it != EndElement(); it++)
		{
			int owner;
			Storage::integer_array v = it->IntegerArrayDV(tag_processors);
			std::sort(v.begin(),v.end());
			if( v.empty() )
			{
				owner = mpirank;
				v.push_back(mpirank);
			}
			else
				owner = std::min(mpirank,v[0]);
			
			it->IntegerDF(tag_owner) = owner;
			
			if( mpirank == owner )
			{
				if( v.size() == 1 )
					it->BulkDF(tag_shared) = Element::Owned;
				else
					it->BulkDF(tag_shared) = Element::Shared;
			}
			else
				it->BulkDF(tag_shared) = Element::Ghost;
		}
		
		ComputeSharedProcs();
		RecomputeParallelStorage(mask);
		AssignGlobalID(mask);
		
		//have to do it for all types
#if defined(USE_PARALLEL_STORAGE)
		for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
		for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)			
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
#endif
	}
	*/
	
	
883
	__INLINE bool point_in_bbox(Storage::real * p, Storage::real bbox[6], unsigned dim, Storage::real eps)
Kirill Terekhov's avatar
Kirill Terekhov committed
884 885 886
	{
		bool ret = true;
		for(unsigned k = 0; k < dim; k++) 
887
			ret &= (p[k] > bbox[k]-eps && p[k] < bbox[dim+k]+eps);
Kirill Terekhov's avatar
Kirill Terekhov committed
888 889 890 891 892 893
		return ret;
	}
	
	__INLINE bool compare_coord(Storage::real * a, Storage::real * b, INMOST_DATA_ENUM_TYPE dim, Storage::real eps)
	{
		for(INMOST_DATA_ENUM_TYPE i = 0; i <  dim; i++)
Kirill Terekhov's avatar
Kirill Terekhov committed
894
			if( ::fabs(a[i]-b[i]) > eps ) return a[i] <= b[i];
Kirill Terekhov's avatar
Kirill Terekhov committed
895 896
		return true;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
897 898 899 900 901

	class MappingComparator
	{
	public: bool operator () (const std::pair<int,int> & a, const std::pair<int,int> & b) {return a.first < b.first;}
	};
902
    
Kirill Terekhov's avatar
Kirill Terekhov committed
903
	
904 905
	
	void Mesh::ResolveShared(bool only_new)
Kirill Terekhov's avatar
Kirill Terekhov committed
906 907
	{
		ENTER_FUNC();
908
		
Kirill Terekhov's avatar
Kirill Terekhov committed
909 910
#if defined(USE_MPI)
		if( m_state == Mesh::Serial ) SetCommunicator(INMOST_MPI_COMM_WORLD);
911
		if( tag_global_id.isValid() ) tag_global_id = DeleteTag(tag_global_id,CELL | EDGE | FACE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
912
		integer dim = GetDimensions();
Kirill Terekhov's avatar
Kirill Terekhov committed
913 914
		int sendsize;
		int mpirank = GetProcessorRank(),mpisize = GetProcessorsNumber();
915
        int rank = mpirank;
Kirill Terekhov's avatar
Kirill Terekhov committed
916 917 918
#if defined(USE_PARALLEL_STORAGE)
		shared_elements.clear();
		ghost_elements.clear();
Kirill Terekhov's avatar
Kirill Terekhov committed
919
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
920 921
		//determine which bboxes i intersect
		dynarray<int,64> procs;
Kirill Terekhov's avatar
Kirill Terekhov committed
922 923 924 925 926
		Storage::real bbox[6]; //local bounding box
		std::vector<Storage::real> bboxs(mpisize*6);
		//Compute local bounding box containing nodes.
		//Will be more convinient to compute (or store)
		//and communicate local octree over all the nodes.
Kirill Terekhov's avatar
Kirill Terekhov committed
927
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
928 929 930 931
		{
			bbox[k] = 1e20;
			bbox[k+dim] = -1e20;
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
932 933 934
#if defined(USE_OMP)
#pragma omp parallel
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
935
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
936
			real bbox0[6];
Kirill Terekhov's avatar
Kirill Terekhov committed
937
			for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
938
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
				bbox0[k] = 1e20;
				bbox0[k+dim] = -1e20;
			}
#if defined(USE_OMP)
#pragma omp for
#endif
			for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
			{
				Node it = NodeByLocalID(nit);
				Storage::real_array arr = it->Coords();
				for(integer k = 0; k < dim; k++)
				{
					if( arr[k] < bbox0[k] ) bbox0[k] = arr[k];
					if( arr[k] > bbox0[k+dim] ) bbox0[k+dim] = arr[k];
				}
			}
#if defined(USE_OMP)
#pragma omp critical
#endif
			{
				for(integer k = 0; k < dim; k++)
				{
					if( bbox0[k] < bbox[k] ) bbox[k] = bbox0[k];
Kirill Terekhov's avatar
Kirill Terekhov committed
962
					if( bbox0[k+dim] > bbox[k+dim] ) bbox[k+dim] = bbox0[k+dim];
Kirill Terekhov's avatar
Kirill Terekhov committed
963
				}
Kirill Terekhov's avatar
Kirill Terekhov committed
964 965
			}
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
966
		REPORT_VAL("dim",dim);
967
		assert(dim*2 <= 6);
Kirill Terekhov's avatar
Kirill Terekhov committed
968
		// write down bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
969
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
970 971 972 973
		{
			REPORT_VAL("min",bbox[k]);
			REPORT_VAL("max",bbox[dim+k]);
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
974
		// communicate bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
975
		REPORT_MPI(MPI_Allgather(&bbox[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,&bboxs[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
976
		// find all processors that i communicate with
Kirill Terekhov's avatar
Kirill Terekhov committed
977 978 979 980
		for(int k = 0; k < mpisize; k++)
			if( k != mpirank )
			{
				bool flag = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
981
				for(integer q = 0; q < dim; q++)
982
					flag &= !((bbox[q]-GetEpsilon() > bboxs[k*dim*2+q+dim]) || (bbox[dim+q]+GetEpsilon() < bboxs[k*dim*2+q]));
Kirill Terekhov's avatar
Kirill Terekhov committed
983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
				if( flag ) procs.push_back(k);
			}
		REPORT_VAL("neighbour processors",procs.size());
		
		//~ if( procs.empty() )
		//~ {
			//~ REPORT_STR("no processors around - all elements are owned");
			//~ for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
			//~ {
				//~ it->IntegerArrayDV(tag_processors).resize(1);
				//~ it->IntegerDF(tag_owner) = it->IntegerDV(tag_processors) = mpirank;
				//~ it->BulkDF(tag_shared) = Element::Owned;
			//~ }
			//~ ComputeSharedProcs();
			//~ RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
			//~ AssignGlobalID(CELL | EDGE | FACE | NODE);
		//~ }
		//~ else
		{
			bool same_boxes = true, same_box;
			for(int k = 0; k < mpisize && same_boxes; k++)
			{
				same_box = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
1006
				for(integer j = 0; j < dim*2; j++)
1007
					same_box &= ::fabs(bbox[j] - bboxs[k*dim*2+j]) < GetEpsilon();
Kirill Terekhov's avatar
Kirill Terekhov committed
1008 1009 1010 1011 1012 1013
				same_boxes &= same_box;
			}
			
			if( same_boxes )
			{
				REPORT_STR("All bounding boxes are the same - assuming that mesh is replicated over all nodes");
Kirill Terekhov's avatar
Kirill Terekhov committed
1014 1015
				//for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
				for(ElementType etype = NODE; etype <= CELL; etype = NextElementType(etype) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1016
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1017 1018 1019 1020 1021 1022
#if defined(USE_OMP)
#pragma omp parallel for
#endif
					for(integer eit = 0; eit < LastLocalID(etype); ++eit) if( isValidElement(etype,eit) )
					{
						Element it = ElementByLocalID(etype,eit);
1023 1024
                        if (only_new && GetMarker(it->GetHandle(),NewMarker()) == false) continue;
                        
Kirill Terekhov's avatar
Kirill Terekhov committed
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
						integer_array arr = it->IntegerArrayDV(tag_processors);
						arr.resize(mpisize);
						for(int k = 0; k < mpisize; k++)
							arr[k] = k;
						it->IntegerDF(tag_owner) = 0;
						if( mpirank == 0 )
							SetStatus(it->GetHandle(),Element::Shared);
						else
							SetStatus(it->GetHandle(),Element::Ghost);
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1035 1036 1037 1038 1039 1040 1041 1042 1043
				}
				ComputeSharedProcs();
				RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
				AssignGlobalID(CELL | EDGE | FACE | NODE);
#if defined(USE_PARALLEL_STORAGE)
				for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1044 1045
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1046
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1047
				for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1048 1049 1050
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1051 1052
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1053
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1054
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
1055 1056 1057
			}
			else
			{
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
                
                if (only_new)
                {
		            for(Mesh::iteratorCell it = BeginCell(); it != EndCell(); it++) if (GetMarker(*it,NewMarker()))
                    {
                        ElementArray<Node> nodes = it->getNodes();
                        for (int i = 0; i < nodes.size(); i++)
                        {
                            nodes[i].SetMarker(NewMarker());
                        }
                    }
                }

Kirill Terekhov's avatar
Kirill Terekhov committed
1071 1072 1073
				double time = Timer();
				Storage::real epsilon = GetEpsilon();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1074
				GeomParam table;
1075
				for(ElementType etype = EDGE; etype <= CELL; etype = NextElementType(etype))
Kirill Terekhov's avatar
Kirill Terekhov committed
1076 1077 1078 1079 1080 1081 1082 1083 1084
					if( !HaveGeometricData(CENTROID,etype) )
						table[CENTROID] |= etype;
				PrepareGeometricData(table);
				
				time = Timer() - time;
			
				REPORT_STR("Prepare geometric data");
				REPORT_VAL("time",time);
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1085 1086 1087 1088 1089
				//for(iteratorNode it = BeginNode(); it != EndNode(); it++)
#if defined(USE_OMP)
#pragma omp parallel for
#endif
				for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1090
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1091
					Node it = NodeByLocalID(nit);
Kirill Terekhov's avatar
Kirill Terekhov committed
1092 1093 1094 1095 1096 1097
					Storage::integer_array arr = it->IntegerArrayDV(tag_processors);
					arr.resize(1);
					arr[0] = mpirank;
				}
			

Kirill Terekhov's avatar
Kirill Terekhov committed
1098
				buffer_type exch_data;
Kirill Terekhov's avatar
Kirill Terekhov committed
1099 1100
				std::vector< INMOST_DATA_REAL_TYPE > unpack_real;
				std::vector< INMOST_DATA_REAL_TYPE > pack_real;
Kirill Terekhov's avatar
Kirill Terekhov committed
1101
				element_set sorted_nodes;
Kirill Terekhov's avatar
Kirill Terekhov committed
1102 1103 1104 1105 1106 1107 1108 1109 1110
				
				sorted_nodes.reserve(NumberOfNodes());
				
				
				time = Timer();
				
				
				for(iteratorNode n = BeginNode(); n != EndNode(); n++)
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1111 1112
					real_array c = n->Coords();
					for(real_array::size_type k = 0; k < procs.size(); k++)
1113
						if( point_in_bbox(c.data(),bboxs.data()+procs[k]*dim*2,dim,GetEpsilon()) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1114
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1115
							sorted_nodes.push_back(*n);
Kirill Terekhov's avatar
Kirill Terekhov committed
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
							break;
						}
				}
				
				time = Timer() - time;
				REPORT_STR("Prepare array of nodes");
				REPORT_VAL("time",time);
				REPORT_VAL("share nodes", sorted_nodes.size());
				REPORT_VAL("total nodes", NumberOfNodes());
				
				
				
				time = Timer();
				if( !sorted_nodes.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1130 1131
					std::sort(sorted_nodes.begin(),sorted_nodes.end(),CentroidComparator(this));
					//qsort(&sorted_nodes[0],sorted_nodes.size(),sizeof(Element *),CompareElementsCCentroid);
Kirill Terekhov's avatar
Kirill Terekhov committed
1132 1133 1134 1135 1136 1137 1138 1139 1140
				time = Timer() - time;
				REPORT_STR("Sort nodes");
				REPORT_VAL("time",time);
				
				
				pack_real.reserve(sorted_nodes.size()*dim);
				
				
				time = Timer();
Kirill Terekhov's avatar
Kirill Terekhov committed
1141
				for(element_set::iterator it = sorted_nodes.begin(); it != sorted_nodes.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1142
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1143
					Storage::real_array arr = RealArrayDF(*it,CoordsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
1144 1145 1146 1147 1148 1149 1150 1151 1152
					pack_real.insert(pack_real.end(),arr.begin(),arr.end());
				}
				time = Timer() - time;
				REPORT_STR("Gather coordinates");
				REPORT_VAL("time",time);
				
				
				time = Timer();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1153
				MPI_Pack_size(static_cast<int>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm,&sendsize);
Kirill Terekhov's avatar
Kirill Terekhov committed
1154 1155
				exch_data.resize(sendsize);
				int position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1156
				if( sendsize > 0 ) MPI_Pack(&pack_real[0],static_cast<INMOST_MPI_SIZE>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,&exch_data[0],static_cast<INMOST_MPI_SIZE>(exch_data.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1157 1158 1159 1160 1161 1162 1163
				
				
				time = Timer() - time;
				REPORT_STR("Pack coordinates");
				REPORT_VAL("time",time);

				
1164
				
Kirill Terekhov's avatar
Kirill Terekhov committed
1165 1166 1167 1168
				{
					std::vector< MPI_Request > send_reqs, recv_reqs;
					exch_buffer_type send_buffs(procs.size()), recv_buffs(procs.size());
					std::vector<int> done;
Kirill Terekhov's avatar
Kirill Terekhov committed
1169
//~ #if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
1170 1171
					//~ unsigned * sendsizeall = shared_space;
					//~ unsigned usend[2] = {sendsize,pack_real.size()};
Kirill Terekhov's avatar
Kirill Terekhov committed
1172
					//~ REPORT_MPI(MPI_Win_fence(0,window)); //start exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1173 1174
					//~ for(unsigned k = 0; k < procs.size(); k++)
						//~ REPORT_MPI(MPI_Put(usend,2,MPI_UNSIGNED,procs[k],mpirank*2,2,MPI_UNSIGNED,window));
Kirill Terekhov's avatar
Kirill Terekhov committed
1175
					//~ REPORT_MPI(MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED,window)); //end exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1176 1177 1178
//~ #else
					std::vector<unsigned> sendsizeall(mpisize*2);
					int pack_size2 = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1179
					unsigned usend[2] = {static_cast<unsigned>(sendsize),static_cast<unsigned>(pack_real.size())};
Kirill Terekhov's avatar
Kirill Terekhov committed
1180
					MPI_Pack_size(2,MPI_UNSIGNED,comm,&pack_size2);
Kirill Terekhov's avatar
Kirill Terekhov committed
1181
					for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1182 1183 1184 1185
					{
						send_buffs[k].first = procs[k];
						send_buffs[k].second.resize(pack_size2);
						position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1186
						MPI_Pack(usend,2,MPI_UNSIGNED,&send_buffs[k].second[0],static_cast<INMOST_MPI_SIZE>(send_buffs[k].second.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1187 1188 1189 1190 1191 1192 1193 1194 1195
						recv_buffs[k].first = procs[k];
						recv_buffs[k].second.resize(pack_size2);
					}
					ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
					while( !(done = FinishRequests(recv_reqs)).empty() )
					{
						for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
						{
							position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1196
							MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&sendsizeall[procs[*qt]*2],2,MPI_UNSIGNED,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1197 1198 1199
						}
					}
					if( !send_reqs.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1200
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
1201
						REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
Kirill Terekhov's avatar
Kirill Terekhov committed
1202
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1203 1204 1205 1206 1207 1208
					//~ REPORT_MPI(MPI_Allgather(usend,2,MPI_UNSIGNED,&sendsizeall[0],2,MPI_UNSIGNED,comm));
//~ #endif
					double time2 = Timer();
					{
						
						
Kirill Terekhov's avatar
Kirill Terekhov committed
1209
						for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
						{
							send_buffs[k].first = procs[k];
							send_buffs[k].second = exch_data;
							recv_buffs[k].first = procs[k];
							recv_buffs[k].second.resize(sendsizeall[procs[k]*2]);
						}
						
						//PrepareReceiveInner(send_buffs,recv_buffs);
						ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
						
						
						
						while( !(done = FinishRequests(recv_reqs)).empty() )
						{
							for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
							{
								time = Timer();
								REPORT_STR("receive node coordinates");
								REPORT_VAL("processor",recv_buffs[*qt].first);
								int count = 0;
								int position = 0;
								unpack_real.resize(sendsizeall[recv_buffs[*qt].first*2+1]);
Kirill Terekhov's avatar
Kirill Terekhov committed
1232
								MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&unpack_real[0],static_cast<INMOST_MPI_SIZE>(unpack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1233 1234 1235 1236
								std::vector<Storage::real>::iterator it1 = pack_real.begin() , it2 = unpack_real.begin();
								while(it1 != pack_real.end() && it2 != unpack_real.end() )
								{
									int res = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1237
									for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1238
										if( ::fabs((*(it1+k))-(*(it2+k))) > epsilon )
Kirill Terekhov's avatar
Kirill Terekhov committed
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
										{
											if( (*(it1+k)) < (*(it2+k)) ) res = -1;
											else res = 1;
											break;
										}
									if( res < 0 ) 
										it1 += dim;
									else if( res > 0 ) 
										it2 += dim;
									else
									{
										count++;
Kirill Terekhov's avatar
Kirill Terekhov committed
1251
										IntegerArrayDV(sorted_nodes[(it1-pack_real.begin())/dim],tag_processors).push_back(recv_buffs[*qt].first);
Kirill Terekhov's avatar
Kirill Terekhov committed
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
										it1 += dim;
										it2 += dim;
									}
								}
								REPORT_VAL("intersected coords",count);
								time = Timer() - time;
								REPORT_STR("Intersect coordinates");
								REPORT_VAL("time",time);
							}
						}
						if( !send_reqs.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1263
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1264
							REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
Kirill Terekhov's avatar
Kirill Terekhov committed
1265
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
1266 1267 1268 1269 1270 1271 1272
					}			
					
					time2 = Timer() - time2;
					REPORT_STR("Intersect all coordinates");
					REPORT_VAL("time",time2);
					
					time = Timer();
Kirill Terekhov's avatar
Kirill Terekhov committed
1273
					Element::Status estat;
Kirill Terekhov's avatar
Kirill Terekhov committed
1274 1275
					for(Mesh::iteratorElement it = BeginElement(NODE); it != EndElement(); it++)
					{
1276
                        if (only_new && GetMarker(*it,NewMarker()) == false) continue;
Kirill Terekhov's avatar
Kirill Terekhov committed
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
						int owner;
						Storage::integer_array v = it->IntegerArrayDV(tag_processors);
						std::sort(v.begin(),v.end());
						if( v.empty() )
						{
							owner = mpirank;
							v.push_back(mpirank);
						}
						else
							owner = std::min(mpirank,v[0]);
						
						it->IntegerDF(tag_owner) = owner;
						
						if( mpirank == owner )
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1292 1293 1294 1295
							if( v.size() == 1 ) 
								estat = Element::Owned;
							else 
								estat = Element::Shared;
Kirill Terekhov's avatar
Kirill Terekhov committed
1296 1297
						}
						else
Kirill Terekhov's avatar
Kirill Terekhov committed
1298 1299
							estat = Element::Ghost;
						SetStatus(*it,estat);
Kirill Terekhov's avatar
Kirill Terekhov committed
1300 1301 1302 1303 1304 1305
					}
					
					ComputeSharedProcs();
					RecomputeParallelStorage(NODE);
					AssignGlobalID(NODE);
					
Kirill Terekhov's avatar
Kirill Terekhov committed
1306
#if defined(USE_PARALLEL_STORAGE)
Kirill Terekhov's avatar
Kirill Terekhov committed
1307 1308 1309
					for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
					{
						if( !it->second[0].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1310 1311
							std::sort(it->second[0].begin(),it->second[0].end(),GlobalIDComparator(this));
							//qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1312 1313 1314 1315
					}
					for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)
					{
						if( !it->second[0].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1316 1317
							std::sort(it->second[0].begin(),it->second[0].end(),GlobalIDComparator(this));
							//qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1318
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1319
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
1320 1321 1322