Commit ea788c00 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

Additional fixes on exchange of elements

parent b62fe3ef
......@@ -405,7 +405,7 @@ namespace INMOST
//as extension of class mesh in limited code span
}
void AdaptiveMesh::CheckParentSet()
void AdaptiveMesh::CheckParentSet(std::string file, int line, TagInteger indicator)
{
ENTER_FUNC();
int err = 0;
......@@ -414,14 +414,16 @@ namespace INMOST
if( parent_set[*it] == InvalidHandle() )
{
REPORT_STR(m->GetProcessorRank() << " parent set not valid on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() << " parent set not valid on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else if( GetHandleElementType(parent_set[*it]) != ESET )
{
REPORT_STR(m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else if( parent_set[*it] != root.GetHandle() )
else if( parent_set[*it] != root.GetHandle() && (!indicator.isValid() || indicator[*it]) )
{
ElementSet set(m,parent_set[*it]);
if( !set.HaveParent() )
......@@ -431,6 +433,11 @@ namespace INMOST
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else
......@@ -443,6 +450,11 @@ namespace INMOST
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" has parent " << ElementTypeName(GetHandleElementType(parent)) << ":" << GetHandleID(parent) <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" has parent " << ElementTypeName(GetHandleElementType(parent)) << ":" << GetHandleID(parent) <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
}
......@@ -452,8 +464,8 @@ namespace INMOST
EXIT_FUNC();
if( err )
{
REPORT_STR(rank << " error in " << __FUNCTION__);
std::cout << rank << " error in " << __FUNCTION__ << std::endl;
REPORT_STR(rank << " error in " << __FUNCTION__ << " " << file << ":" << line);
std::cout << rank << " error in " << __FUNCTION__ << " " << file << ":" << line << std::endl;
exit(-1);
}
......@@ -490,7 +502,7 @@ namespace INMOST
//m->CheckCentroids(__FILE__,__LINE__);
m->ExchangeData(hanging_nodes,CELL | FACE,0);
//m->CheckCentroids(__FILE__,__LINE__);
CheckParentSet();
//CheckParentSet(Tag());
EXIT_BLOCK();
//ENTER_BLOCK();
......@@ -1231,8 +1243,11 @@ namespace INMOST
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( indicator[set] != 0 )
set.SynchronizeSetElements();
if( set.GetName().substr(0,3) == "AM_" )
{
if( indicator[set] != 0 && !set.Empty() )
set.SynchronizeSetElements();
}
}
EXIT_BLOCK();
m->ExchangeMarked();
......@@ -1251,19 +1266,34 @@ namespace INMOST
//m->CheckCentroids(__FILE__,__LINE__);
EXIT_BLOCK();
//m->Barrier();
ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( indicator[set] != 0 )
set.SynchronizeSetParents();
if( set.GetName().substr(0,3) == "AM_" )
{
//int imax = -1, imin = INT_MAX;
//for(ElementSet::iterator jt = set.Begin(); jt != set.End(); ++jt)
//{
// imax = std::max(imax,indicator[*jt]);
// imin = std::min(imin,indicator[*jt]);
//}
//std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax;
//if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName();
//std::cout << std::endl;
if( indicator[set] != 0 )
set.SynchronizeSetParents();
}
}
EXIT_BLOCK();
//m->Barrier();
//std::cout << m->GetProcessorRank() << " call exchange marked" << std::endl;
m->ExchangeMarked();
CheckParentSet();
//std::cout << m->GetProcessorRank() << " finish exchange marked" << std::endl;
//m->Barrier();
CheckParentSet(__FILE__,__LINE__,indicator);
//std::fstream fout("sets"+std::to_string(m->GetProcessorRank())+".txt",std::ios::out);
//for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it)
......@@ -1278,7 +1308,7 @@ namespace INMOST
m->BeginModification();
while(schedule_counter)
{
CheckParentSet();
CheckParentSet(__FILE__,__LINE__,indicator);
//CheckParentSet();
//fout << "schedule_counter " << schedule_counter << std::endl;
//unite cells
......
......@@ -16,7 +16,7 @@ namespace INMOST
int size;
/// Prepare sets for coarsements.
/// Do not do this in constructor, since mesh may contain no cells.
void CheckParentSet();
void CheckParentSet(std::string file, int line, TagInteger indicator);
void PrepareSet();
//void PrintSetLocal(std::string offset, ElementSet it, std::stringstream& ss);
//void SynchronizeIndicated(TagInteger& indicator);
......
......@@ -69,6 +69,8 @@ int main(int argc, char ** argv)
for(int k = 0; k < 15; ++k)
{
m.ClearFile();
int numref;
int refcnt = 0;
......
......@@ -485,7 +485,10 @@ namespace INMOST
/// 3. Should correctly account for order of edges (may be implemented through CheckEdgeOrder, FixEdgeOrder).
void Connect (const HandleType * adjacent, INMOST_DATA_ENUM_TYPE num) const;
/// Update geometric data for element, calls RecomputeGeometricData from Mesh.
void UpdateGeometricData () const;
void UpdateGeometricData () const;
/// Marks element to be sent to remote processors that current processor don't belong to.
/// Call Mesh::ExchangeMarked to perform the exchange.
void SendTo (std::set<Storage::integer> & procs) const;
};
__INLINE const Element & InvalidElement() {static Element ret(NULL,InvalidHandle()); return ret;}
......@@ -2323,6 +2326,7 @@ namespace INMOST
int & GetFuncID () {return func_id;}
std::fstream & GetStream ();
std::ostream & WriteTab (std::ostream & f);
void ClearFile ();
void FinalizeFile ();
static void AtExit (void);
#endif
......@@ -2919,6 +2923,7 @@ namespace INMOST
/// @param op operation, one of SYNC_BIT_SET, SYNC_BIT_OR, SYNC_BIT_XOR, SYNC_BIT_AND
void SynchronizeMarker (MarkerType marker, ElementType mask, SyncBitOp op);
//for debug
void Barrier ();
void BeginSequentialCode();
void EndSequentialCode ();
//iterator.cpp::::::::::::::::::::::::::::::::::::::::::::::::::
......
......@@ -1273,7 +1273,20 @@ namespace INMOST
GetMeshLink()->SetGeometricType(GetHandle(),t);
}
void Element::SendTo(std::set<Storage::integer> & procs) const
{
if( GetMeshLink()->GetMeshState() != Mesh::Serial )
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
Storage::integer_array sendto = IntegerArray(GetMeshLink()->SendtoTag());
std::sort(sendto.begin(),sendto.end());
std::vector<Storage::integer> tmp1(procs.size()),tmp2;
tmp1.resize(std::set_difference(procs.begin(),procs.end(),set_procs.begin(),set_procs.end(),tmp1.begin())-tmp1.begin());
tmp2.resize(tmp1.size()+sendto.size());
tmp2.resize(std::set_union(tmp1.begin(),tmp1.end(),sendto.begin(),sendto.end(),tmp2.begin())-tmp2.begin());
sendto.replace(sendto.begin(),sendto.end(),tmp2.begin(),tmp2.end());
}
}
}
#endif
......@@ -1768,47 +1768,20 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
Storage::integer_array set_procs = IntegerArray(m->ProcessorsTag());
Storage::integer_array elem_procs;
std::set<Storage::integer> send_set;
std::vector<Storage::integer> send_procs, temp;
std::vector<Storage::integer>::iterator itr;
//Storage::integer_array elem_procs;
std::set<Storage::integer> send_set(set_procs.begin(),set_procs.end());
//for(iterator it = Begin(); it != End(); ++it)
//{
// elem_procs = m->IntegerArray(*it,m->ProcessorsTag());
// send_set.insert(elem_procs.begin(),elem_procs.end());
//}
for(iterator it = Begin(); it != End(); ++it)
{
elem_procs = m->IntegerArray(*it,m->ProcessorsTag());
send_procs.resize(set_procs.size());
send_procs.resize(std::set_difference(set_procs.begin(),set_procs.end(),
elem_procs.begin(),elem_procs.end(),
send_procs.begin())-send_procs.begin());
if( !send_procs.empty() )
{
Storage::integer_array sendto = m->IntegerArray(*it,m->SendtoTag());
send_set.insert(send_procs.begin(),send_procs.end());
std::sort(sendto.begin(),sendto.end());
temp.resize(sendto.size()+send_procs.size());
temp.resize(std::set_union(sendto.begin(),sendto.end(),
send_procs.begin(),send_procs.end(),
temp.begin())-temp.begin());
sendto.resize(temp.size());
for(unsigned k = 0; k < sendto.size(); ++k) sendto[k] = temp[k];
}
}
Storage::integer_array sendto = IntegerArray(m->SendtoTag());
sendto.resize(send_set.size());
unsigned k = 0;
for(std::set<Storage::integer>::iterator it = send_set.begin();
it != send_set.end(); ++it) sendto[k++] = *it;
it->SendTo(send_set);
}
}
void ElementSet::SetSendTo(std::set<Storage::integer> & procs, char dir)
{
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
Storage::integer_array sendto = IntegerArray(GetMeshLink()->SendtoTag());
sendto.resize(procs.size());
sendto.resize(std::set_difference(procs.begin(),procs.end(),
set_procs.begin(),set_procs.end(),
sendto.begin())-sendto.begin());
}
SendTo(procs);
if( (dir & 1) && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
......@@ -1816,7 +1789,7 @@ namespace INMOST
}
if( (dir & 2) && HaveParent() )
{
GetParent();
GetParent()->SetSendTo(procs,dir & 2); // don't let parent to go downwards
}
}
void ElementSet::CollectProcessors(std::set<Storage::integer> & procs, char dir)
......@@ -1825,12 +1798,12 @@ namespace INMOST
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
procs.insert(set_procs.begin(),set_procs.end());
}
if( dir & 1 && HaveChild() )
if( (dir & 1) && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
it.CollectProcessors(procs,dir & 1); //don't let children to go upwards
}
if( dir & 2 && HaveParent() )
if( (dir & 2) && HaveParent() )
{
GetParent().CollectProcessors(procs,dir & 2); //don't let parent to go downwards
}
......@@ -1842,8 +1815,8 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set,1);
SetSendTo(send_set,1);
CollectProcessors(send_set,1); //collect procs from children
if( !send_set.empty() ) SetSendTo(send_set,1);
}
}
......@@ -1853,8 +1826,11 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set,2);
SetSendTo(send_set,2);
CollectProcessors(send_set,0); //don't collect procs from parents
//std::cout << GetMeshLink()->GetProcessorRank() << " SynchronizeSetParents " << GetName() << " procs ";
//for(std::set<Storage::integer>::iterator it = send_set.begin(); it != send_set.end(); ++it) std::cout << *it << " ";
if( !send_set.empty() ) SetSendTo(send_set,2);
//std::cout << std::endl;
}
}
......
......@@ -192,7 +192,14 @@ namespace INMOST
}
#endif
ClearFile();
allocated_meshes.push_back(this);
}
void Mesh::ClearFile()
{
#if defined(USE_PARALLEL_WRITE_TIME)
if( out_time.is_open() ) out_time.close();
num_exchanges = 0;
std::stringstream temp;
temp << "time_" << GetProcessorRank() << ".xml";
......@@ -203,8 +210,7 @@ namespace INMOST
tab = 1;
func_id = 0;
#endif
allocated_meshes.push_back(this);
}
}
Mesh::Mesh()
:TagManager(), Storage(NULL,ComposeHandle(MESH,0))
......
......@@ -2985,6 +2985,12 @@ namespace INMOST
assert( k < array_size_recv.size() );
assert( array_size_recv[k] < elements[i].size() );
eit = elements[i].begin() + array_size_recv[k++];
//if( select && !GetMarker(*eit,select) )
//{
// std::cout << __FILE__ << ":" << __LINE__ << " unpack " << tag.GetTagName() << " from " << source;
// std::cout << " element " << ElementTypeName(GetHandleElementType(*eit)) << ":" << GetHandleID(*eit);
// std::cout << " no marker" << std::endl;
//}
assert( !select || GetMarker(*eit,select) ); //if fires then very likely that marker was not synchronized
if( tag.GetDataType() == DATA_REFERENCE )
{
......@@ -3023,6 +3029,15 @@ namespace INMOST
assert( k < array_size_recv.size() );
assert( array_size_recv[k] < elements[i].size() );
eit = elements[i].begin() + array_size_recv[k++];
if( select && !GetMarker(*eit,select) )
{
std::cout << __FILE__ << ":" << __LINE__ << " " << GetProcessorRank();
std::cout << " unpack " << tag.GetTagName() << " from " << source;
std::cout << " offset " << array_size_recv[k-1];
std::cout << " element " << ElementTypeName(GetHandleElementType(*eit)) << ":" << GetHandleID(*eit);
if( GetHandleElementType(*eit) == ESET ) std::cout << " " << ElementSet(this,*eit).GetName();
std::cout << " no marker" << std::endl;
}
assert( !select || GetMarker(*eit,select) ); //if fires then very likely that marker was not synchronized
if( tag.GetDataType() == DATA_REFERENCE )
{
......@@ -3664,7 +3679,7 @@ namespace INMOST
ENTER_FUNC();
for(proc_elements_by_type::iterator it = pselems.begin(); it != pselems.end(); ++it)
{
}
EXIT_FUNC();
}
......@@ -3752,39 +3767,30 @@ namespace INMOST
{
ElementSet set(this,selems[4][i]);
// looking to child, sibling and parent
if( set.HaveChild() )
{
if( !set.GetChild().GetMarker(busy) && !set.GetChild().Hidden() )
{
Storage::integer_array procs = set.GetChild().IntegerArrayDV(ProcessorsTag());
if( std::binary_search(procs.begin(),procs.end(),destination) )
{
selems[4].push_back(set.GetChild().GetHandle());
set.GetChild().SetMarker(busy);
}
}
}
if( set.HaveSibling() )
if( set.HaveParent() )
{
if( !set.GetSibling().GetMarker(busy) && !set.GetSibling().Hidden() )
if( !set.GetParent().GetMarker(busy) && !set.GetParent().Hidden() )
{
Storage::integer_array procs = set.GetSibling().IntegerArrayDV(ProcessorsTag());
Storage::integer_array procs = set.GetParent().IntegerArrayDV(ProcessorsTag());
if( std::binary_search(procs.begin(),procs.end(),destination) )
{
selems[4].push_back(set.GetSibling().GetHandle());
set.GetSibling().SetMarker(busy);
selems[4].push_back(set.GetParent().GetHandle());
set.GetParent().SetMarker(busy);
}
}
}
if( set.HaveParent() )
if( set.HaveChild() )
{
if( !set.GetParent().GetMarker(busy) && !set.GetParent().Hidden() )
for(ElementSet jt = set.GetChild(); jt.isValid(); jt = jt.GetSibling() )
{
//Storage::integer_array procs = set.GetParent().IntegerArrayDV(ProcessorsTag());
//if( std::binary_search(procs.begin(),procs.end(),destination) )
if( !jt.GetMarker(busy) && !jt.Hidden() )
{
selems[4].push_back(set.GetParent().GetHandle());
set.GetParent().SetMarker(busy);
Storage::integer_array procs = jt.IntegerArrayDV(ProcessorsTag());
if( std::binary_search(procs.begin(),procs.end(),destination) )
{
selems[4].push_back(jt.GetHandle());
jt.SetMarker(busy);
}
}
}
}
......@@ -3842,58 +3848,58 @@ namespace INMOST
position = static_cast<int>(buffer.size());
new_size = 0;
MPI_Pack_size(1 ,INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(1 ,MPI_INT ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[0].size()*dim),INMOST_MPI_DATA_REAL_TYPE ,comm,&temp); new_size += temp;
if( HaveGlobalID(NODE) )
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[0].size()),INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
buffer.resize(position+new_size);
num = static_cast<INMOST_DATA_ENUM_TYPE>(selems[0].size());
MPI_Pack(&num,1,INMOST_MPI_DATA_ENUM_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
int marked_for_data = 0, marked_shared = 0;
k = 0;
std::vector<Storage::real> coords(selems[0].size()*dim);
for(element_set::iterator it = selems[0].begin(); it != selems[0].end(); it++)
{
k = 0;
std::vector<Storage::real> coords(selems[0].size()*dim);
for(element_set::iterator it = selems[0].begin(); it != selems[0].end(); it++)
Storage::real_array c = RealArray(*it,tag_coords);
//REPORT_VAL("packed coords",k << " " << c[0] << " " << c[1] << " " << c[2]);
for(integer j = 0; j < dim; j++) coords[k*GetDimensions()+j] = c[j];
Storage::integer & owner = IntegerDF(*it,tag_owner);
{
Storage::real_array c = RealArray(*it,tag_coords);
//REPORT_VAL("packed coords",k << " " << c[0] << " " << c[1] << " " << c[2]);
for(integer j = 0; j < dim; j++) coords[k*GetDimensions()+j] = c[j];
Storage::integer & owner = IntegerDF(*it,tag_owner);
if( owner == mpirank )
{
if( owner == mpirank )
Storage::integer_array proc = IntegerArrayDV(*it,tag_processors);
Storage::integer_array::iterator ip = std::lower_bound(proc.begin(),proc.end(),destination);
if( ip == proc.end() || (*ip) != destination ) proc.insert(ip,destination);
if( GetStatus(*it) != Element::Shared)
{
Storage::integer_array proc = IntegerArrayDV(*it,tag_processors);
Storage::integer_array::iterator ip = std::lower_bound(proc.begin(),proc.end(),destination);
if( ip == proc.end() || (*ip) != destination ) proc.insert(ip,destination);
if( GetStatus(*it) != Element::Shared)
{
//if( GetStatus(*it) != Element::Owned )
// std::cout << "node " << GetHandleID(*it) << " " << Element::StatusName(GetStatus(*it)) << std::endl;
//assert(GetStatus(*it) == Element::Owned);
++marked_shared;
SetStatus(*it, Element::Shared);
}
//if( GetStatus(*it) != Element::Owned )
// std::cout << "node " << GetHandleID(*it) << " " << Element::StatusName(GetStatus(*it)) << std::endl;
//assert(GetStatus(*it) == Element::Owned);
++marked_shared;
SetStatus(*it, Element::Shared);
}
}
//TODO: 45
if( owner != destination )
{
SetMarker(*it,pack_tags_mrk);
//TODO 46 old
// pack_tags[0].push_back(*it);
++marked_for_data;
}
if( HaveGlobalID(NODE) )
{
global_ids.push_back(Integer(*it,GlobalIDTag()));
//REPORT_VAL("packed global_id",global_ids.back());
}
k++;
}
if( !coords.empty() )
MPI_Pack(&coords[0] ,static_cast<INMOST_MPI_SIZE>(selems[0].size()*dim),INMOST_MPI_DATA_REAL_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( HaveGlobalID(NODE) && !global_ids.empty() )
MPI_Pack(&global_ids[0],static_cast<INMOST_MPI_SIZE>(selems[0].size()) ,INMOST_MPI_DATA_INTEGER_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
//TODO: 45
if( owner != destination )
{
SetMarker(*it,pack_tags_mrk);
//TODO 46 old
// pack_tags[0].push_back(*it);
++marked_for_data;
}
if( HaveGlobalID(NODE) )
{
global_ids.push_back(Integer(*it,GlobalIDTag()));
//REPORT_VAL("packed global_id",global_ids.back());
}
k++;
}
MPI_Pack(&num,1,INMOST_MPI_DATA_ENUM_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
MPI_Pack(&marked_for_data,1,MPI_INT,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !coords.empty() )
MPI_Pack(&coords[0] ,static_cast<INMOST_MPI_SIZE>(selems[0].size()*dim),INMOST_MPI_DATA_REAL_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( HaveGlobalID(NODE) && !global_ids.empty() )
MPI_Pack(&global_ids[0],static_cast<INMOST_MPI_SIZE>(selems[0].size()) ,INMOST_MPI_DATA_INTEGER_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
REPORT_VAL("total marked for data", marked_for_data << " / " << selems[0].size());
REPORT_VAL("total marked as shared", marked_shared << " / " << selems[0].size());
buffer.resize(position);
......@@ -3969,12 +3975,15 @@ namespace INMOST
REPORT_VAL("number of edge nodes",num);
REPORT_VAL("total marked for data", marked_for_data << " / " << selems[1].size());
REPORT_VAL("total marked as shared", marked_shared << " / " << selems[1].size());
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(1+selems[1].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
MPI_Pack_size(1 ,INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(1 ,MPI_INT ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[1].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
//MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[1].size()) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
buffer.resize(position+new_size);
temp = static_cast<INMOST_DATA_ENUM_TYPE>(selems[1].size());
MPI_Pack(&temp,1,INMOST_MPI_DATA_ENUM_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
MPI_Pack(&marked_for_data,1,MPI_INT,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_size.empty() ) MPI_Pack(&low_conn_size[0],static_cast<INMOST_MPI_SIZE>(selems[1].size()),INMOST_MPI_DATA_ENUM_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_nums.empty() ) MPI_Pack(&low_conn_nums[0],static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
buffer.resize(position);
......@@ -4039,12 +4048,15 @@ namespace INMOST
REPORT_VAL("number of face edges",num);
REPORT_VAL("total marked for data", marked_for_data << " / " << selems[2].size());
REPORT_VAL("total marked as shared", marked_shared << " / " << selems[2].size());
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(1+selems[2].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(1) ,INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(1) ,MPI_INT ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[2].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
//MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[2].size()) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
buffer.resize(position+new_size);
temp = static_cast<INMOST_DATA_ENUM_TYPE>(selems[2].size());
MPI_Pack(&temp,1,INMOST_MPI_DATA_ENUM_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
MPI_Pack(&marked_for_data,1,MPI_INT,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_size.empty() ) MPI_Pack(&low_conn_size[0],static_cast<INMOST_MPI_SIZE>(selems[2].size()),INMOST_MPI_DATA_ENUM_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_nums.empty() ) MPI_Pack(&low_conn_nums[0],static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
buffer.resize(position);
......@@ -4131,6 +4143,7 @@ namespace INMOST
REPORT_VAL("total marked for data", marked_for_data << " / " << selems[3].size());
REPORT_VAL("total marked as shared", marked_shared << " / " << selems[3].size());
MPI_Pack_size(1 ,INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(1 ,MPI_INT ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[3].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,comm,&temp); new_size += temp;
MPI_Pack_size(static_cast<INMOST_MPI_SIZE>(selems[3].size()),INMOST_MPI_DATA_ENUM_TYPE ,comm,&temp); new_size += temp;
......@@ -4139,6 +4152,7 @@ namespace INMOST
buffer.resize(position+new_size);
temp = static_cast<INMOST_DATA_ENUM_TYPE>(selems[3].size());
MPI_Pack(&temp,1,INMOST_MPI_DATA_ENUM_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
MPI_Pack(&marked_for_data,1,MPI_INT,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_size.empty() ) MPI_Pack(&low_conn_size[0] ,static_cast<INMOST_MPI_SIZE>(selems[3].size()),INMOST_MPI_DATA_ENUM_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !low_conn_nums.empty() ) MPI_Pack(&low_conn_nums[0] ,static_cast<INMOST_MPI_SIZE>(num) ,INMOST_MPI_DATA_INTEGER_TYPE,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
if( !high_conn_size.empty() ) MPI_Pack(&high_conn_size[0],static_cast<INMOST_MPI_SIZE>(selems[3].size()),INMOST_MPI_DATA_ENUM_TYPE ,&buffer[0],static_cast<INMOST_MPI_SIZE>(buffer.size()),&position,comm);
......@@ -4151,17 +4165,16 @@ namespace INMOST
// pack esets
ENTER_BLOCK();
{
std::vector<INMOST_DATA_ENUM_TYPE> low_conn_size(selems[4].size());