Commit ea788c00 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

Additional fixes on exchange of elements

parent b62fe3ef
......@@ -405,7 +405,7 @@ namespace INMOST
//as extension of class mesh in limited code span
}
void AdaptiveMesh::CheckParentSet()
void AdaptiveMesh::CheckParentSet(std::string file, int line, TagInteger indicator)
{
ENTER_FUNC();
int err = 0;
......@@ -414,14 +414,16 @@ namespace INMOST
if( parent_set[*it] == InvalidHandle() )
{
REPORT_STR(m->GetProcessorRank() << " parent set not valid on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() << " parent set not valid on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else if( GetHandleElementType(parent_set[*it]) != ESET )
{
REPORT_STR(m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else if( parent_set[*it] != root.GetHandle() )
else if( parent_set[*it] != root.GetHandle() && (!indicator.isValid() || indicator[*it]) )
{
ElementSet set(m,parent_set[*it]);
if( !set.HaveParent() )
......@@ -431,6 +433,11 @@ namespace INMOST
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
else
......@@ -443,6 +450,11 @@ namespace INMOST
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" has parent " << ElementTypeName(GetHandleElementType(parent)) << ":" << GetHandleID(parent) <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
std::cout << m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" has parent " << ElementTypeName(GetHandleElementType(parent)) << ":" << GetHandleID(parent) <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++;
}
}
......@@ -452,8 +464,8 @@ namespace INMOST
EXIT_FUNC();
if( err )
{
REPORT_STR(rank << " error in " << __FUNCTION__);
std::cout << rank << " error in " << __FUNCTION__ << std::endl;
REPORT_STR(rank << " error in " << __FUNCTION__ << " " << file << ":" << line);
std::cout << rank << " error in " << __FUNCTION__ << " " << file << ":" << line << std::endl;
exit(-1);
}
......@@ -490,7 +502,7 @@ namespace INMOST
//m->CheckCentroids(__FILE__,__LINE__);
m->ExchangeData(hanging_nodes,CELL | FACE,0);
//m->CheckCentroids(__FILE__,__LINE__);
CheckParentSet();
//CheckParentSet(Tag());
EXIT_BLOCK();
//ENTER_BLOCK();
......@@ -1231,9 +1243,12 @@ namespace INMOST
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( indicator[set] != 0 )
if( set.GetName().substr(0,3) == "AM_" )
{
if( indicator[set] != 0 && !set.Empty() )
set.SynchronizeSetElements();
}
}
EXIT_BLOCK();
m->ExchangeMarked();
......@@ -1251,19 +1266,34 @@ namespace INMOST
//m->CheckCentroids(__FILE__,__LINE__);
EXIT_BLOCK();
//m->Barrier();
ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( set.GetName().substr(0,3) == "AM_" )
{
//int imax = -1, imin = INT_MAX;
//for(ElementSet::iterator jt = set.Begin(); jt != set.End(); ++jt)
//{
// imax = std::max(imax,indicator[*jt]);
// imin = std::min(imin,indicator[*jt]);
//}
//std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax;
//if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName();
//std::cout << std::endl;
if( indicator[set] != 0 )
set.SynchronizeSetParents();
}
}
EXIT_BLOCK();
//m->Barrier();
//std::cout << m->GetProcessorRank() << " call exchange marked" << std::endl;
m->ExchangeMarked();
CheckParentSet();
//std::cout << m->GetProcessorRank() << " finish exchange marked" << std::endl;
//m->Barrier();
CheckParentSet(__FILE__,__LINE__,indicator);
//std::fstream fout("sets"+std::to_string(m->GetProcessorRank())+".txt",std::ios::out);
//for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it)
......@@ -1278,7 +1308,7 @@ namespace INMOST
m->BeginModification();
while(schedule_counter)
{
CheckParentSet();
CheckParentSet(__FILE__,__LINE__,indicator);
//CheckParentSet();
//fout << "schedule_counter " << schedule_counter << std::endl;
//unite cells
......
......@@ -16,7 +16,7 @@ namespace INMOST
int size;
/// Prepare sets for coarsements.
/// Do not do this in constructor, since mesh may contain no cells.
void CheckParentSet();
void CheckParentSet(std::string file, int line, TagInteger indicator);
void PrepareSet();
//void PrintSetLocal(std::string offset, ElementSet it, std::stringstream& ss);
//void SynchronizeIndicated(TagInteger& indicator);
......
......@@ -70,6 +70,8 @@ int main(int argc, char ** argv)
for(int k = 0; k < 15; ++k)
{
m.ClearFile();
int numref;
int refcnt = 0;
do
......
......@@ -486,6 +486,9 @@ namespace INMOST
void Connect (const HandleType * adjacent, INMOST_DATA_ENUM_TYPE num) const;
/// Update geometric data for element, calls RecomputeGeometricData from Mesh.
void UpdateGeometricData () const;
/// Marks element to be sent to remote processors that current processor don't belong to.
/// Call Mesh::ExchangeMarked to perform the exchange.
void SendTo (std::set<Storage::integer> & procs) const;
};
__INLINE const Element & InvalidElement() {static Element ret(NULL,InvalidHandle()); return ret;}
......@@ -2323,6 +2326,7 @@ namespace INMOST
int & GetFuncID () {return func_id;}
std::fstream & GetStream ();
std::ostream & WriteTab (std::ostream & f);
void ClearFile ();
void FinalizeFile ();
static void AtExit (void);
#endif
......@@ -2919,6 +2923,7 @@ namespace INMOST
/// @param op operation, one of SYNC_BIT_SET, SYNC_BIT_OR, SYNC_BIT_XOR, SYNC_BIT_AND
void SynchronizeMarker (MarkerType marker, ElementType mask, SyncBitOp op);
//for debug
void Barrier ();
void BeginSequentialCode();
void EndSequentialCode ();
//iterator.cpp::::::::::::::::::::::::::::::::::::::::::::::::::
......
......@@ -1273,7 +1273,20 @@ namespace INMOST
GetMeshLink()->SetGeometricType(GetHandle(),t);
}
void Element::SendTo(std::set<Storage::integer> & procs) const
{
if( GetMeshLink()->GetMeshState() != Mesh::Serial )
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
Storage::integer_array sendto = IntegerArray(GetMeshLink()->SendtoTag());
std::sort(sendto.begin(),sendto.end());
std::vector<Storage::integer> tmp1(procs.size()),tmp2;
tmp1.resize(std::set_difference(procs.begin(),procs.end(),set_procs.begin(),set_procs.end(),tmp1.begin())-tmp1.begin());
tmp2.resize(tmp1.size()+sendto.size());
tmp2.resize(std::set_union(tmp1.begin(),tmp1.end(),sendto.begin(),sendto.end(),tmp2.begin())-tmp2.begin());
sendto.replace(sendto.begin(),sendto.end(),tmp2.begin(),tmp2.end());
}
}
}
#endif
......@@ -1768,47 +1768,20 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
Storage::integer_array set_procs = IntegerArray(m->ProcessorsTag());
Storage::integer_array elem_procs;
std::set<Storage::integer> send_set;
std::vector<Storage::integer> send_procs, temp;
std::vector<Storage::integer>::iterator itr;
//Storage::integer_array elem_procs;
std::set<Storage::integer> send_set(set_procs.begin(),set_procs.end());
//for(iterator it = Begin(); it != End(); ++it)
//{
// elem_procs = m->IntegerArray(*it,m->ProcessorsTag());
// send_set.insert(elem_procs.begin(),elem_procs.end());
//}
for(iterator it = Begin(); it != End(); ++it)
{
elem_procs = m->IntegerArray(*it,m->ProcessorsTag());
send_procs.resize(set_procs.size());
send_procs.resize(std::set_difference(set_procs.begin(),set_procs.end(),
elem_procs.begin(),elem_procs.end(),
send_procs.begin())-send_procs.begin());
if( !send_procs.empty() )
{
Storage::integer_array sendto = m->IntegerArray(*it,m->SendtoTag());
send_set.insert(send_procs.begin(),send_procs.end());
std::sort(sendto.begin(),sendto.end());
temp.resize(sendto.size()+send_procs.size());
temp.resize(std::set_union(sendto.begin(),sendto.end(),
send_procs.begin(),send_procs.end(),
temp.begin())-temp.begin());
sendto.resize(temp.size());
for(unsigned k = 0; k < sendto.size(); ++k) sendto[k] = temp[k];
}
}
Storage::integer_array sendto = IntegerArray(m->SendtoTag());
sendto.resize(send_set.size());
unsigned k = 0;
for(std::set<Storage::integer>::iterator it = send_set.begin();
it != send_set.end(); ++it) sendto[k++] = *it;
it->SendTo(send_set);
}
}
void ElementSet::SetSendTo(std::set<Storage::integer> & procs, char dir)
{
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
Storage::integer_array sendto = IntegerArray(GetMeshLink()->SendtoTag());
sendto.resize(procs.size());
sendto.resize(std::set_difference(procs.begin(),procs.end(),
set_procs.begin(),set_procs.end(),
sendto.begin())-sendto.begin());
}
SendTo(procs);
if( (dir & 1) && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
......@@ -1816,7 +1789,7 @@ namespace INMOST
}
if( (dir & 2) && HaveParent() )
{
GetParent();
GetParent()->SetSendTo(procs,dir & 2); // don't let parent to go downwards
}
}
void ElementSet::CollectProcessors(std::set<Storage::integer> & procs, char dir)
......@@ -1825,12 +1798,12 @@ namespace INMOST
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
procs.insert(set_procs.begin(),set_procs.end());
}
if( dir & 1 && HaveChild() )
if( (dir & 1) && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
it.CollectProcessors(procs,dir & 1); //don't let children to go upwards
}
if( dir & 2 && HaveParent() )
if( (dir & 2) && HaveParent() )
{
GetParent().CollectProcessors(procs,dir & 2); //don't let parent to go downwards
}
......@@ -1842,8 +1815,8 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set,1);
SetSendTo(send_set,1);
CollectProcessors(send_set,1); //collect procs from children
if( !send_set.empty() ) SetSendTo(send_set,1);
}
}
......@@ -1853,8 +1826,11 @@ namespace INMOST
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set,2);
SetSendTo(send_set,2);
CollectProcessors(send_set,0); //don't collect procs from parents
//std::cout << GetMeshLink()->GetProcessorRank() << " SynchronizeSetParents " << GetName() << " procs ";
//for(std::set<Storage::integer>::iterator it = send_set.begin(); it != send_set.end(); ++it) std::cout << *it << " ";
if( !send_set.empty() ) SetSendTo(send_set,2);
//std::cout << std::endl;
}
}
......
......@@ -192,7 +192,14 @@ namespace INMOST
}
#endif
ClearFile();
allocated_meshes.push_back(this);
}
void Mesh::ClearFile()
{
#if defined(USE_PARALLEL_WRITE_TIME)
if( out_time.is_open() ) out_time.close();
num_exchanges = 0;
std::stringstream temp;
temp << "time_" << GetProcessorRank() << ".xml";
......@@ -203,7 +210,6 @@ namespace INMOST
tab = 1;
func_id = 0;
#endif
allocated_meshes.push_back(this);
}
Mesh::Mesh()
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment