Commit c75eb406 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

add more timers for parallel adaptation code

parent be438831
This diff is collapsed.
......@@ -833,43 +833,53 @@ namespace INMOST
}
else if( i == myconn + 3 )
{
//~ std::vector<HandleType> tmp;
MarkerType mrk = mesh->CreatePrivateMarker();
adj_type const & hc = mesh->HighConn(GetHandle());
for(adj_type::size_type it = 0; it < hc.size(); it++)
{
adj_type const & ihc = mesh->HighConn(hc[it]);
for(adj_type::size_type jt = 0; jt < ihc.size(); jt++)
{
adj_type const & jhc = mesh->HighConn(ihc[jt]);
for(adj_type::size_type kt = 0; kt < jhc.size(); kt++)
if( !mesh->GetPrivateMarker(jhc[kt],mrk) )
{
result.push_back(jhc[kt]);
mesh->SetPrivateMarker(jhc[kt],mrk);
}
}
//~ if( !mesh->GetPrivateMarker(ihc[jt],mrk) )
{
//~ mesh->SetPrivateMarker(ihc[jt],mrk);
//~ tmp.push_back(ihc[jt]);
adj_type const & jhc = mesh->HighConn(ihc[jt]);
for(adj_type::size_type kt = 0; kt < jhc.size(); kt++)
if( !mesh->GetPrivateMarker(jhc[kt],mrk) )
{
result.push_back(jhc[kt]);
mesh->SetPrivateMarker(jhc[kt],mrk);
}
}
}
//~ if(!tmp.empty()) mesh->RemPrivateMarkerArray(&tmp[0],tmp.size(),mrk);
result.RemPrivateMarker(mrk);
mesh->ReleasePrivateMarker(mrk);
}
else if( i == myconn - 3 )
{
//~ std::vector<HandleType> tmp;
MarkerType mrk = mesh->CreatePrivateMarker();
adj_type const & lc = mesh->LowConn(GetHandle());
for(adj_type::size_type it = 0; it < lc.size(); it++)
{
adj_type const & ilc = mesh->LowConn(lc[it]);
for(adj_type::size_type jt = 0; jt < ilc.size(); jt++)
{
adj_type const & jlc = mesh->LowConn(ilc[jt]);
for(adj_type::size_type kt = 0; kt < jlc.size(); kt++)
if( !mesh->GetPrivateMarker(jlc[kt],mrk) )
{
result.push_back(jlc[kt]);
mesh->SetPrivateMarker(jlc[kt],mrk);
}
}
//~ if( !mesh->GetPrivateMarker(ilc[jt],mrk) )
{
//~ mesh->SetPrivateMarker(ilc[jt],mrk);
//~ tmp.push_back(ilc[jt]);
adj_type const & jlc = mesh->LowConn(ilc[jt]);
for(adj_type::size_type kt = 0; kt < jlc.size(); kt++)
if( !mesh->GetPrivateMarker(jlc[kt],mrk) )
{
result.push_back(jlc[kt]);
mesh->SetPrivateMarker(jlc[kt],mrk);
}
}
}
//~ if(!tmp.empty()) mesh->RemPrivateMarkerArray(&tmp[0],tmp.size(),mrk);
result.RemPrivateMarker(mrk);
mesh->ReleasePrivateMarker(mrk);
}
......@@ -934,6 +944,7 @@ namespace INMOST
}
else if( i == myconn + 3 )
{
//~ std::vector<HandleType> tmp;
MarkerType mrk = mesh->CreatePrivateMarker();
adj_type const & hc = mesh->HighConn(GetHandle());
for(adj_type::size_type it = 0; it < hc.size(); it++) if( !mesh->GetMarker(hc[it],hm) )
......@@ -941,22 +952,29 @@ namespace INMOST
adj_type const & ihc = mesh->HighConn(hc[it]);
for(adj_type::size_type jt = 0; jt < ihc.size(); jt++) if( !mesh->GetMarker(ihc[jt],hm) )
{
adj_type const & jhc = mesh->HighConn(ihc[jt]);
for(adj_type::size_type kt = 0; kt < jhc.size(); kt++) if( !mesh->GetMarker(jhc[kt],hm) )
//~ if( !mesh->GetPrivateMarker(ihc[jt],mrk) )
{
if( !mesh->GetPrivateMarker(jhc[kt],mrk) )
//~ mesh->SetPrivateMarker(ihc[jt],mrk);
//~ tmp.push_back(ihc[jt]);
adj_type const & jhc = mesh->HighConn(ihc[jt]);
for(adj_type::size_type kt = 0; kt < jhc.size(); kt++) if( !mesh->GetMarker(jhc[kt],hm) )
{
result.push_back(jhc[kt]);
mesh->SetPrivateMarker(jhc[kt],mrk);
if( !mesh->GetPrivateMarker(jhc[kt],mrk) )
{
result.push_back(jhc[kt]);
mesh->SetPrivateMarker(jhc[kt],mrk);
}
}
}
}
}
//~ if(!tmp.empty()) mesh->RemPrivateMarkerArray(&tmp[0],tmp.size(),mrk);
result.RemPrivateMarker(mrk);
mesh->ReleasePrivateMarker(mrk);
}
else if( i == myconn - 3 )
{
//~ std::vector<HandleType> tmp;
MarkerType mrk = mesh->CreatePrivateMarker();
adj_type const & lc = mesh->LowConn(GetHandle());
for(adj_type::size_type it = 0; it < lc.size(); it++) if( !mesh->GetMarker(lc[it],hm) )
......@@ -964,17 +982,23 @@ namespace INMOST
adj_type const & ilc = mesh->LowConn(lc[it]);
for(adj_type::size_type jt = 0; jt != ilc.size(); jt++) if( !mesh->GetMarker(ilc[jt],hm) )
{
adj_type const & jlc = mesh->LowConn(ilc[jt]);
for(adj_type::size_type kt = 0; kt < jlc.size(); kt++) if( !mesh->GetMarker(jlc[kt],hm) )
//~ if( !mesh->GetPrivateMarker(ilc[jt],mrk) )
{
if( !mesh->GetPrivateMarker(jlc[kt],mrk) )
//~ mesh->SetPrivateMarker(ilc[jt],mrk);
//~ tmp.push_back(ilc[jt]);
adj_type const & jlc = mesh->LowConn(ilc[jt]);
for(adj_type::size_type kt = 0; kt < jlc.size(); kt++) if( !mesh->GetMarker(jlc[kt],hm) )
{
result.push_back(jlc[kt]);
mesh->SetPrivateMarker(jlc[kt],mrk);
if( !mesh->GetPrivateMarker(jlc[kt],mrk) )
{
result.push_back(jlc[kt]);
mesh->SetPrivateMarker(jlc[kt],mrk);
}
}
}
}
}
//~ if(!tmp.empty()) mesh->RemPrivateMarkerArray(&tmp[0],tmp.size(),mrk);
result.RemPrivateMarker(mrk);
mesh->ReleasePrivateMarker(mrk);
}
......
......@@ -7,10 +7,19 @@
// incident_matrix class should measure for minimal volume,
// possibly check and update from projects/OctreeCutcell/octgrid.cpp
#if defined(USE_PARALLEL_WRITE_TIME)
__INLINE std::string NameSlash(std::string input)
{
for(size_t l = input.size(); l > 0; --l)
if( input[l-1] == '/' || input[l-1] == '\\' )
return std::string(input.c_str() + l);
return input;
}
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
#define ENTER_BLOCK() { double btime = Timer(); WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << ":" << NameSlash(__FILE__) << ":" << __LINE__ << "\" id=\"func" << GetFuncID()++ << "\">" << std::endl; Enter();
#define EXIT_BLOCK() WriteTab(out_time) << "<TIME>" << Timer() - btime << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#else
......@@ -18,6 +27,8 @@
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
#define ENTER_BLOCK()
#define EXIT_BLOCK()
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE() {}
#endif
......@@ -1539,6 +1550,7 @@ namespace INMOST
void Mesh::ApplyModification()
{
ENTER_FUNC();
ENTER_BLOCK();
for(Mesh::iteratorTag it = BeginTag(); it != EndTag(); ++it)
{
if( it->GetDataType() == DATA_REFERENCE )
......@@ -1573,6 +1585,7 @@ namespace INMOST
}
}
}
EXIT_BLOCK();
//need to gather the set of deleted elements
/*//old approach
ElementSet erase = CreateSet("TEMPORARY_ERASE_SET").first;
......@@ -1588,6 +1601,7 @@ namespace INMOST
erase->BulkDF(SetComparatorTag()) = ElementSet::HANDLE_COMPARATOR;
*/
ENTER_BLOCK();
//for(integer jt = 0; jt < LastLocalID(ESET); ++jt) if( isValidElementSet(jt) )
for(Mesh::iteratorSet it = BeginSet(); it != EndSet(); it++)
{
......@@ -1618,6 +1632,8 @@ namespace INMOST
hide_element = temp_hide_element;
//it->Subtract(erase); //old approach
}
EXIT_BLOCK();
ENTER_BLOCK();
#if defined(USE_PARALLEL_STORAGE)
for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
for(int i = 0; i < 5; i++)
......@@ -1642,6 +1658,7 @@ namespace INMOST
it->second[i].resize(k);
}
#endif
EXIT_BLOCK();
//Destroy(erase);//old approach
EXIT_FUNC();
}
......@@ -1663,12 +1680,13 @@ namespace INMOST
}
std::cout << GetProcessorRank() << " before resolve shared new " << n << " hidden " << h << " both " << hn << std::endl;
*/
ENTER_BLOCK();
CheckSetLinks(__FILE__,__LINE__);
ResolveSets();
CheckSetLinks(__FILE__,__LINE__);
ResolveShared(true);
CheckSetLinks(__FILE__,__LINE__);
EXIT_BLOCK();
//ReportParallelStorage();
//CheckCentroids(__FILE__,__LINE__);
/*
......@@ -1684,8 +1702,10 @@ namespace INMOST
std::cout << GetProcessorRank() << " before exchange ghost new " << n << " hidden " << h << " both " << hn << std::endl;
*/
//std::cout << "layers " << Integer(GetHandle(),tag_layers) << " bridge " << ElementTypeName(ElementType(Integer(GetHandle(),tag_bridge))) << std::endl;
ENTER_BLOCK();
if( Integer(GetHandle(),tag_layers) )
ExchangeGhost(Integer(GetHandle(),tag_layers),Integer(GetHandle(),tag_bridge));//,NewMarker()); //TODO!!!!
EXIT_BLOCK();
//ReportParallelStorage();
//CheckCentroids(__FILE__,__LINE__);
......@@ -1728,6 +1748,7 @@ namespace INMOST
}
*/
MarkerType nm = new_element;
ENTER_BLOCK();
new_element = 0;
for(ElementType etype = ESET; etype >= NODE; etype = PrevElementType(etype))
{
......@@ -1740,6 +1761,7 @@ namespace INMOST
}
}
new_element = nm;
EXIT_BLOCK();
/*
for(ElementType etype = FACE; etype >= NODE; etype = PrevElementType(etype))
{
......@@ -1760,11 +1782,13 @@ namespace INMOST
new_element = 0;
//This should be done in ResolveModification
ElementType have_global_id = NONE;
ENTER_BLOCK();
if( GlobalIDTag().isValid() )
{
for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype))
if( GlobalIDTag().isDefined(etype) ) have_global_id |= etype;
}
EXIT_BLOCK();
if( have_global_id ) AssignGlobalID(have_global_id);
EXIT_FUNC();
}
......
......@@ -26,7 +26,7 @@ bool allow_pack_by_gids = false;
#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
#endif // USE_MPI
#if defined(USE_PARALLEL_WRITE_TIME)
__INLINE std::string NameSlash(std::string input)
{
for(size_t l = input.size(); l > 0; --l)
......@@ -34,8 +34,6 @@ __INLINE std::string NameSlash(std::string input)
return std::string(input.c_str() + l);
return input;
}
#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
......@@ -453,7 +451,9 @@ namespace INMOST
if( tag_bridge.isValid() ) bridge = ElementType(Integer(GetHandle(),tag_bridge));
TagIntegerArray tag = CreateTag("TEMPORARY_OWNER_DISTANCE_TAG",DATA_INTEGER,CELL,CELL,2);
std::queue<HandleType> cells_queue;
double t, tadj = 0;
ENTER_BLOCK();
REPORT_STR("Collect cells into queue");
//TODO: compute graph distance only in new elements!!!!
// Push nearest to owned cell into queue
for(integer k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
......@@ -465,7 +465,9 @@ namespace INMOST
tag[c][0] = -1;
tag[c][1] = INT_MAX;
//ElementArray<Cell> cells = c.NeighbouringCells();
t = Timer();
ElementArray<Cell> cells = c.BridgeAdjacencies2Cell(bridge);
tadj += Timer()-t;
for(INMOST_DATA_ENUM_TYPE l = 0; l < cells.size(); l++) if (cells[l].GetStatus() == Element::Owned)
{
cells_queue.push(c.GetHandle());
......@@ -478,6 +480,7 @@ namespace INMOST
EXIT_BLOCK();
int cur_dist = 1; // For assert
ENTER_BLOCK();
REPORT_STR("graph distance calculation");
while (!cells_queue.empty())
{
Cell c(this,cells_queue.front());
......@@ -485,7 +488,9 @@ namespace INMOST
if (cur_dist > tag[c][1]) assert(0);
if (cur_dist < tag[c][1]) cur_dist++;
//ElementArray<Cell> cells = c.NeighbouringCells();
t = Timer();
ElementArray<Cell> cells = c.BridgeAdjacencies2Cell(bridge);
tadj += Timer()-t;
for(INMOST_DATA_ENUM_TYPE l = 0; l < cells.size(); l++) if (cells[l].GetStatus() != Element::Owned)
{
assert( cells[l].GetStatus() == Element::Shared || cells[l].GetStatus() == Element::Ghost );
......@@ -498,6 +503,7 @@ namespace INMOST
}
}
EXIT_BLOCK();
REPORT_VAL("adjacency requests", tadj);
//static int equiv = 0;
//std::string file;
......@@ -515,11 +521,11 @@ namespace INMOST
//std::cout << "Save " << file << std::endl;
//Save(file);
//equiv++;
ENTER_BLOCK();
CheckGhostSharedCount(__FILE__,__LINE__);
CheckCentroids(__FILE__,__LINE__);
EXIT_BLOCK();
ENTER_BLOCK();
//TagInteger tag_new_owner = CreateTag("TEMPORARY_NEW_OWNER",DATA_INTEGER,CELL,NONE,1);
//for(int k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
//{
......@@ -531,7 +537,7 @@ namespace INMOST
// }
//}
//ExchangeData(tag_new_owner,CELL,0);
ENTER_BLOCK();
for(integer k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
{
Cell c = CellByLocalID(k);
......@@ -560,9 +566,10 @@ namespace INMOST
}
}
}
EXIT_BLOCK();
//DeleteTag(tag_new_owner);
//ExchangeData(tag_owner,CELL,0);
ENTER_BLOCK();
RecomputeParallelStorage(CELL);
CheckGhostSharedCount(__FILE__,__LINE__,CELL);
EXIT_BLOCK();
......@@ -621,7 +628,9 @@ namespace INMOST
}
EXIT_BLOCK();
RecomputeParallelStorage(etype);
ENTER_BLOCK();
CheckGhostSharedCount(__FILE__,__LINE__,etype);
EXIT_BLOCK();
}
DeleteTag(new_owner);
//ComputeSharedProcs();
......@@ -1509,6 +1518,7 @@ namespace INMOST
bbox[k] = 1e20;
bbox[k+dim] = -1e20;
}
ENTER_BLOCK();
#if defined(USE_OMP)
#pragma omp parallel
#endif
......@@ -1543,6 +1553,7 @@ namespace INMOST
}
}
}
EXIT_BLOCK();
REPORT_VAL("dim",dim);
assert(dim*2 <= 6);
// write down bounding boxes
......@@ -1552,8 +1563,11 @@ namespace INMOST
REPORT_VAL("max",bbox[dim+k]);
}
// communicate bounding boxes
ENTER_BLOCK();
REPORT_MPI(MPI_Allgather(&bbox[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,&bboxs[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,comm));
EXIT_BLOCK();
// find all processors that i communicate with
ENTER_BLOCK();
for(int k = 0; k < mpisize; k++)
if( k != mpirank )
{
......@@ -1562,6 +1576,7 @@ namespace INMOST
flag &= !((bbox[q]-GetEpsilon() > bboxs[(size_t)k*dim*2+q+dim]) || (bbox[dim+q]+GetEpsilon() < bboxs[(size_t)k*dim*2+q]));
if( flag ) procs.push_back(k);
}
EXIT_BLOCK();
REPORT_VAL("neighbour processors",procs.size());
EXIT_BLOCK();
//~ if( procs.empty() )
......@@ -1582,6 +1597,7 @@ namespace INMOST
ENTER_BLOCK();
{
bool same_boxes = true, same_box;
ENTER_BLOCK();
for(int k = 0; k < mpisize && same_boxes; k++)
{
same_box = true;
......@@ -1589,6 +1605,7 @@ namespace INMOST
same_box &= ::fabs(bbox[j] - bboxs[(size_t)k*dim*2+j]) < GetEpsilon();
same_boxes &= same_box;
}
EXIT_BLOCK();
//if( same_boxes )
if( false )
......@@ -1658,6 +1675,7 @@ namespace INMOST
Storage::real epsilon = GetEpsilon();
/*
GeomParam table;
//if( !only_new )
......@@ -1672,7 +1690,7 @@ namespace INMOST
REPORT_STR("Prepare geometric data");
EXIT_BLOCK();
}
*/
//for(iteratorNode it = BeginNode(); it != EndNode(); it++)
ENTER_BLOCK();
......@@ -1723,9 +1741,9 @@ namespace INMOST
REPORT_STR("Sort nodes");
EXIT_BLOCK();
ENTER_BLOCK();
pack_real.reserve(sorted_nodes.size()*dim);
EXIT_BLOCK();
ENTER_BLOCK();
for(element_set::iterator it = sorted_nodes.begin(); it != sorted_nodes.end(); it++)
......@@ -1738,18 +1756,18 @@ namespace INMOST
//~ int position = 0;
ENTER_BLOCK();
ENTER_BLOCK();
pack_data_vector(exch_data,pack_real,GetCommunicator());
EXIT_BLOCK();
REPORT_STR("Pack coordinates");
EXIT_BLOCK();
ENTER_BLOCK();
{
exch_reqs_type send_reqs;
exch_recv_reqs_type recv_reqs;
......@@ -1775,10 +1793,15 @@ namespace INMOST
{
//~ position = 0;
//~ MPI_Unpack(&recv_buffs[*qt].second[0],static_cast<INMOST_MPI_SIZE>(recv_buffs[*qt].second.size()),&position,&sendsizeall[procs[*qt]*2],2,MPI_UNSIGNED,comm);
ENTER_BLOCK();
REPORT_VAL("processor",procs[*qt]);
size_t buf_pos = 0;
unpack_data(recv_buffs[*qt].second,buf_pos,sendsizeall[procs[*qt]],GetCommunicator());
EXIT_BLOCK();
}
}
EXIT_BLOCK();
ENTER_BLOCK();
if( !send_reqs.empty() )
{
REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
......@@ -1843,7 +1866,8 @@ namespace INMOST
REPORT_STR("Intersect coordinates");
}
}
EXIT_BLOCK();
ENTER_BLOCK();
if( !send_reqs.empty() )
{
REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
......@@ -1885,9 +1909,11 @@ namespace INMOST
}
EXIT_BLOCK();
//ComputeSharedProcs();
ENTER_BLOCK();
RecomputeParallelStorage(NODE);
AssignGlobalID(NODE);
ReportParallelStorage();
EXIT_BLOCK();
ENTER_BLOCK();
#if defined(USE_PARALLEL_STORAGE)
for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
......@@ -2024,6 +2050,7 @@ namespace INMOST
{
message_send.clear();
message_send.push_back(0);
ENTER_BLOCK();
for(Mesh::iteratorElement it = BeginElement(current_mask); it != EndElement(); it++)
{
if (only_new && !it->GetMarker(new_lc)) continue;
......@@ -2049,13 +2076,14 @@ namespace INMOST
elements[m].push_back(*it);
}
}
EXIT_BLOCK();
REPORT_VAL("gathered elements",elements[m].size());
ENTER_BLOCK();
send_buffs[m].first = *p;
pack_data_vector(send_buffs[m].second,message_send,GetCommunicator());
recv_buffs[m].first = *p;
EXIT_BLOCK();
}
}
......@@ -2103,6 +2131,7 @@ namespace INMOST
if( message_recv[m].empty() ) continue;
integer num_remote_elements = message_recv[m][pos++];
REPORT_VAL("number of remote elements",num_remote_elements);
ENTER_BLOCK();
for(integer i = 0; i < num_remote_elements; i++)
{
integer conn_size = message_recv[m][pos++], flag = 1;
......@@ -2138,6 +2167,7 @@ namespace INMOST
}
}
REPORT_VAL("number of unpacked remote elements",remote_elements.size());
EXIT_BLOCK();
//if( !remote_elements.empty() )
//{
// REPORT_VAL("first",remote_elements.front());
......@@ -2145,7 +2175,9 @@ namespace INMOST
// REPORT_VAL("last",remote_elements.back());
// REPORT_VAL("last type",ElementTypeName(GetHandleElementType(remote_elements.back())));
//}
ENTER_BLOCK();
std::sort(remote_elements.begin(),remote_elements.end());
EXIT_BLOCK();
REPORT_VAL("original elements size",elements[m].size());
//if( !elements[m].empty() )
//{
......@@ -2154,16 +2186,18 @@ namespace INMOST
// REPORT_VAL("last",elements[m].back());
// REPORT_VAL("last type",ElementTypeName(GetHandleElementType(elements[m].back())));
//}
ENTER_BLOCK();
std::sort(elements[m].begin(),elements[m].end());
EXIT_BLOCK();
element_set result;
element_set::iterator set_end;
ENTER_BLOCK();
result.resize(elements[m].size());
set_end = std::set_difference(elements[m].begin(),elements[m].end(),remote_elements.begin(),remote_elements.end(), result.begin());
result.resize(set_end-result.begin());
EXIT_BLOCK();
REPORT_VAL("set difference size",result.size());
ENTER_BLOCK();
//elements in result are wrongly marked as ghost
for(element_set::iterator qt = result.begin(); qt != result.end(); qt++)
{
......@@ -2171,6 +2205,7 @@ namespace INMOST
integer_array::iterator find = std::lower_bound(pr.begin(),pr.end(),*p);
pr.erase(find);
}
EXIT_BLOCK();
}
......@@ -2204,10 +2239,12 @@ namespace INMOST
}
EXIT_BLOCK();
RecomputeParallelStorage(current_mask);
ENTER_BLOCK();
if( !send_reqs.empty() )
{
REPORT_MPI(MPI_Waitall(static_cast<INMOST_MPI_SIZE>(send_reqs.size()),&send_reqs[0],MPI_STATUSES_IGNORE));
}
EXIT_BLOCK();
AssignGlobalID(current_mask);
ReportParallelStorage();
integer num = ElementNum(current_mask);
......@@ -2232,16 +2269,21 @@ namespace INMOST
if( only_new ) ReleaseMarker(new_lc,EDGE|FACE|CELL);
}
EXIT_BLOCK();
//if( !only_new )
RemoveGeometricData(table);
//ENTER_BLOCK();
//RemoveGeometricData(table);
//EXIT_BLOCK();
}
}
EXIT_BLOCK();
ENTER_BLOCK();
ReportParallelStorage();
CheckGhostSharedCount(__FILE__,__LINE__);
EquilibrateGhost();
CheckGhostSharedCount(__FILE__,__LINE__);
CheckProcsSorted(__FILE__,__LINE__);
EXIT_BLOCK();
//RemoveGhost();
#else //USE_MPI
AssignGlobalID(CELL | FACE | EDGE | NODE);
......@@ -2445,22 +2487,21 @@ namespace INMOST
for(ElementType mask = ESET; mask >= NODE; mask = PrevElementType(mask))
{
//std::cout << GetProcessorRank() << " start " << ElementTypeName(mask) << std::endl;
//if( mask & CELL )
//{
INMOST_DATA_ENUM_TYPE cnt = 0;
ENTER_BLOCK();
for(const HandleType * it = ghost; it != ghost + num; it++)
if( (GetHandleElementType(*it) & mask) && GetStatus(*it) ==