Commit 286eae95 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

Maintain parent connections of sets when sending sets to other processors

parent 2e63185b
......@@ -421,6 +421,32 @@ namespace INMOST
REPORT_STR(m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
err++;
}
else if( parent_set[*it] != root.GetHandle() )
{
ElementSet set(m,parent_set[*it]);
if( !set.HaveParent() )
{
REPORT_STR(m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
err++;
}
else
{
HandleType parent = set.GetParent().GetHandle();
if( GetHandleElementType(parent) != ESET )
{
REPORT_STR(m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) <<
" has parent " << ElementTypeName(GetHandleElementType(parent)) << ":" << GetHandleID(parent) <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]);
err++;
}
}
}
}
err = m->Integrate(err);
EXIT_FUNC();
......@@ -1206,13 +1232,12 @@ namespace INMOST
{
ElementSet set = m->EsetByLocalID(it);
if( indicator[set] != 0 )
{
set.SynchronizeSetElements();
if( set.HaveParent() ) set.GetParent().SynchronizeSetTree();
}
}
EXIT_BLOCK();
m->ExchangeMarked();
//CheckParentSet();
ENTER_BLOCK();
//m->CheckCentroids(__FILE__,__LINE__);
......@@ -1224,8 +1249,21 @@ namespace INMOST
//m->CheckCentroids(__FILE__,__LINE__);
m->ExchangeData(hanging_nodes,CELL | FACE,0);
//m->CheckCentroids(__FILE__,__LINE__);
CheckParentSet();
EXIT_BLOCK();
ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( indicator[set] != 0 )
set.SynchronizeSetParents();
}
EXIT_BLOCK();
m->ExchangeMarked();
CheckParentSet();
//std::fstream fout("sets"+std::to_string(m->GetProcessorRank())+".txt",std::ios::out);
//for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it)
......@@ -1241,6 +1279,7 @@ namespace INMOST
while(schedule_counter)
{
CheckParentSet();
//CheckParentSet();
//fout << "schedule_counter " << schedule_counter << std::endl;
//unite cells
//should find and set hanging nodes on faces
......
......@@ -970,8 +970,14 @@ namespace INMOST
__INLINE ElementSet & self () {return *this;}
__INLINE const ElementSet & self () const {return *this;}
private:
void CollectProcessors (std::set<Storage::integer> & procs);
void SetSendTo(std::set<Storage::integer> & procs);
/// Compute union of processors on elements.
/// @param procs Set of processors to fill.
/// @param dir Directon for tree traversal, 1 - downwards, 2, upwards, 3 - both directions
void CollectProcessors (std::set<Storage::integer> & procs, char dir);
/// Fill Mesh::SendtoTag() to send sets to processors from procs set that currently don't have this set
/// @param procs Set of processors to send the set to.
/// @param dir Directon for tree traversal, 1 - downwards, 2, upwards, 3 - both directions
void SetSendTo(std::set<Storage::integer> & procs, char dir);
public:
/// Get name of the set
std::string GetName() const;
......@@ -1256,7 +1262,13 @@ namespace INMOST
/// Asks all the children to be sent to other processors.
/// Call ExchangeMarked afterwards.
/// @see Mesh::ExchangeMarked
void SynchronizeSetTree();
void SynchronizeSetChildren();
/// Asks all the parents upwards to be sent to other processors.
/// This function does not ask children of the parents to be synchronized,
/// for this traverse to the upppermost parent and call ElementSet::SynchronizeSetChildren
/// Call ExchangeMarked afterwards.
/// @see Mesh::ExchangeMarked
void SynchronizeSetParents();
};
__INLINE const ElementSet & InvalidElementSet() {static ElementSet ret(NULL,InvalidHandle()); return ret;}
......
......@@ -1799,7 +1799,7 @@ namespace INMOST
it != send_set.end(); ++it) sendto[k++] = *it;
}
}
void ElementSet::SetSendTo(std::set<Storage::integer> & procs)
void ElementSet::SetSendTo(std::set<Storage::integer> & procs, char dir)
{
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
......@@ -1809,36 +1809,55 @@ namespace INMOST
set_procs.begin(),set_procs.end(),
sendto.begin())-sendto.begin());
}
if( HaveChild() )
if( (dir & 1) && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
it->SetSendTo(procs);
it->SetSendTo(procs, dir & 1); // don't let children to go upwards
}
if( (dir & 2) && HaveParent() )
{
GetParent();
}
}
void ElementSet::CollectProcessors(std::set<Storage::integer> & procs)
void ElementSet::CollectProcessors(std::set<Storage::integer> & procs, char dir)
{
{
Storage::integer_array set_procs = IntegerArray(GetMeshLink()->ProcessorsTag());
procs.insert(set_procs.begin(),set_procs.end());
}
if( HaveChild() )
if( dir & 1 && HaveChild() )
{
for(ElementSet it = GetChild(); it != InvalidElementSet(); it = it.GetSibling() )
it.CollectProcessors(procs);
it.CollectProcessors(procs,dir & 1); //don't let children to go upwards
}
if( dir & 2 && HaveParent() )
{
GetParent().CollectProcessors(procs,dir & 2); //don't let parent to go downwards
}
}
void ElementSet::SynchronizeSetTree()
void ElementSet::SynchronizeSetChildren()
{
Mesh * m = GetMeshLink();
if( m->GetMeshState() == Mesh::Serial ) return;
if( GetMeshLink()->GetMeshState() == Mesh::Serial ) return;
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set);
SetSendTo(send_set);
CollectProcessors(send_set,1);
SetSendTo(send_set,1);
}
}
void ElementSet::SynchronizeSetParents()
{
if( GetMeshLink()->GetMeshState() == Mesh::Serial ) return;
if( GetStatus() != Element::Owned )
{
std::set<Storage::integer> send_set;
CollectProcessors(send_set,2);
SetSendTo(send_set,2);
}
}
}
#endif
......@@ -2088,6 +2088,7 @@ namespace INMOST
}
std::cout << GetProcessorRank() << " before resolve shared new " << n << " hidden " << h << " both " << hn << std::endl;
*/
ResolveSets();
ResolveShared(true);
//ReportParallelStorage();
//CheckCentroids(__FILE__,__LINE__);
......@@ -2120,7 +2121,7 @@ namespace INMOST
}
std::cout << GetProcessorRank() << " after exchange ghost new " << n << " hidden " << h << " both " << hn << std::endl;
*/
ResolveSets();
//ReportParallelStorage();
//CheckCentroids(__FILE__,__LINE__);
//exit(-1);
......
......@@ -2137,21 +2137,31 @@ namespace INMOST
ENTER_FUNC();
#if defined(USE_MPI)
int mpirank = GetProcessorRank();
Tag tag_delete = CreateTag("TEMPORARY_DELETE_GHOST_ELEMENTS_TAG",DATA_INTEGER,CELL | FACE | EDGE | NODE,CELL | FACE | EDGE | NODE);
Tag tag_delete = CreateTag("TEMPORARY_DELETE_GHOST_ELEMENTS_TAG",DATA_INTEGER,ESET | CELL | FACE | EDGE | NODE, ESET | CELL | FACE | EDGE | NODE);
element_set delete_elements;
#if defined(USE_PARALLEL_STORAGE)
proc_elements del_shared, del_ghost;
#endif //USE_PARALLEL_STORAGE
for(ElementType mask = CELL; mask >= NODE; mask = PrevElementType(mask))
for(ElementType mask = ESET; mask >= NODE; mask = PrevElementType(mask))
{
//std::cout << GetProcessorRank() << " start " << ElementTypeName(mask) << std::endl;
if( mask & CELL )
//if( mask & CELL )
//{
int cnt = 0;
for(const HandleType * it = ghost; it != ghost + num; it++)
if( (GetHandleElementType(*it) & mask) && GetStatus(*it) == Element::Ghost )
{
Integer(*it,tag_delete) = mpirank;
cnt++;
}
if( mask & (ESET|CELL) )
{
for(const HandleType * it = ghost; it != ghost + num; it++)
if( GetStatus(*it) == Element::Ghost ) Integer(*it,tag_delete) = mpirank;
cnt = Integrate(cnt);
if( !cnt ) continue;
}
else
//}
if( mask & (FACE|EDGE|NODE) )
{
for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
{
......@@ -3446,6 +3456,12 @@ namespace INMOST
for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
{
Element::Status estat = it->GetStatus();
if( it->Hidden() )
{
err++;
REPORT_STR(GetProcessorRank() << " " << __FILE__ << ":" << __LINE__ << "Adding hidden element " << ElementTypeName(mask));
std::cout << GetProcessorRank() << " " << __FILE__ << ":" << __LINE__ << "Adding hidden element " << ElementTypeName(mask) << std::endl;
}
if( estat == Element::Shared )
{
Storage::integer_array v = it->IntegerArrayDV(tag_processors);
......@@ -3754,8 +3770,8 @@ namespace INMOST
{
if( !set.GetParent().GetMarker(busy) && !set.GetParent().Hidden() )
{
Storage::integer_array procs = set.GetParent().IntegerArrayDV(ProcessorsTag());
if( std::binary_search(procs.begin(),procs.end(),destination) )
//Storage::integer_array procs = set.GetParent().IntegerArrayDV(ProcessorsTag());
//if( std::binary_search(procs.begin(),procs.end(),destination) )
{
selems[4].push_back(set.GetParent().GetHandle());
set.GetParent().SetMarker(busy);
......@@ -5755,12 +5771,13 @@ namespace INMOST
ENTER_FUNC();
#if defined(USE_MPI)
REPORT_STR("sort shared elements")
double time = Timer();
ENTER_BLOCK();
for(parallel_storage::iterator it = shared.begin(); it != shared.end(); it++)
{
REPORT_VAL("processor",it->first);
for(int i = 0; i < 4; i++) if( mask & ElementTypeFromDim(i) )
{
ENTER_BLOCK();
REPORT_STR(ElementTypeName(ElementTypeFromDim(i)) << " have global id " << (!tag_global_id.isValid() ? "invalid" : (tag_global_id.isDefined(ElementTypeFromDim(i))?"YES":"NO")));
if( !it->second[i].empty() )
{
......@@ -5770,11 +5787,13 @@ namespace INMOST
std::sort(it->second[i].begin(),it->second[i].end(),CentroidComparator(this));
}
REPORT_VAL(ElementTypeName(mask & ElementTypeFromDim(i)),it->second[i].size());
EXIT_BLOCK();
}
// ESET sort
if (mask & ElementTypeFromDim(4))
{
ENTER_BLOCK();
if( !it->second[4].empty() )
{
if(HaveGlobalID(ElementTypeFromDim(4)))
......@@ -5791,17 +5810,18 @@ namespace INMOST
// }
}
REPORT_VAL(ElementTypeName(mask & ElementTypeFromDim(4)),it->second[4].size());
EXIT_BLOCK();
}
}
time = Timer() - time;
REPORT_VAL("time",time);
EXIT_BLOCK();
REPORT_STR("sort ghost elements")
time = Timer();
ENTER_BLOCK();
for(parallel_storage::iterator it = ghost.begin(); it != ghost.end(); it++)
{
REPORT_VAL("processor",it->first);
for(int i = 0; i < 4; i++) if( mask & ElementTypeFromDim(i) )
{
ENTER_BLOCK();
REPORT_STR(ElementTypeName(ElementTypeFromDim(i)) << " have global id " << (!tag_global_id.isValid() ? "invalid" : (tag_global_id.isDefined(ElementTypeFromDim(i))?"YES":"NO")));
if( !it->second[i].empty() )
{
......@@ -5811,11 +5831,13 @@ namespace INMOST
std::sort(it->second[i].begin(),it->second[i].end(),CentroidComparator(this));
}
REPORT_VAL(ElementTypeName(mask & ElementTypeFromDim(i)),it->second[i].size());
EXIT_BLOCK();
}
// ESET sort
if (mask & ElementTypeFromDim(4))
{
ENTER_BLOCK();
if( !it->second[4].empty() )
{
if(HaveGlobalID(ElementTypeFromDim(4)))
......@@ -5832,10 +5854,10 @@ namespace INMOST
// }
}
REPORT_VAL(ElementTypeName(mask & ElementTypeFromDim(4)),it->second[4].size());
EXIT_BLOCK();
}
}
time = Timer() - time;
REPORT_VAL("time",time);
EXIT_BLOCK();
#else //USE_MPI and USE_PARALLEL_STORAGE
(void) ghost; (void) shared; (void) mask;
#endif //USE_MPI and USE_PARALLEL_STORAGE
......@@ -6412,10 +6434,10 @@ namespace INMOST
std::vector<int> size_recv(mpisize);
int size_send, size_recv_all = 0, prealloc = 0;
ENTER_BLOCK();
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it)
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it) if( !Hidden(it->second) )
prealloc += it->first.size()+1;
set_names_send.reserve(prealloc);
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it)
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it) if( !Hidden(it->second) )
{
set_names_send.insert(set_names_send.end(),it->first.begin(),it->first.end());
set_names_send.push_back('\0');
......@@ -6451,7 +6473,7 @@ namespace INMOST
// Change status for self sets
//for(Mesh::iteratorSet set = BeginSet(); set != EndSet(); ++set)
/*
ENTER_BLOCK();
for(std::map<std::string,std::vector<int> >::iterator it = map_names.begin(); it != map_names.end(); ++it)
{
......@@ -6468,11 +6490,14 @@ namespace INMOST
for(std::vector<int>::iterator jt = it->second.begin(); jt != it->second.end(); ++jt) REPORT_STR("proc " << *jt);
}
EXIT_BLOCK();
*/
ENTER_BLOCK();
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it)
REPORT_VAL("number of sets in set_search: ", set_search.size());
REPORT_VAL("number of sets in mesh: ", NumberOfSets());
REPORT_VAL("number of sets in map_names: ", map_names.size());
for(std::map<std::string,HandleType>::iterator it = set_search.begin(); it != set_search.end(); ++it) if( !Hidden(it->second) )
{
REPORT_STR("\"" << it->first << "\" size " << it->first.size() << " handle " << it->second);
//REPORT_STR("\"" << it->first << "\" size " << it->first.size() << " handle " << it->second);
std::vector<int> & procs = map_names[it->first];
//if( procs.empty() )
// std::cout << "no procs for " << it->first << std::endl;
......@@ -6505,8 +6530,16 @@ namespace INMOST
else
SetStatus(it->second, Element::Ghost);
}
REPORT_VAL("owner ",IntegerDF(it->second, tag_owner));
REPORT_VAL("status ",Element::StatusName(GetStatus(it->second)));
std::stringstream pstr;
pstr << "name " << it->first;
pstr << " handle " << it->second;
pstr << " owner " << IntegerDF(it->second, tag_owner);
pstr << " status " << Element::StatusName(GetStatus(it->second));
pstr << " procs";
if( Hidden(it->second) ) pstr << " hidden";
else pstr << " not hidden";
for (int i = 0; i < arr.size(); i++) pstr << " " << arr[i];
REPORT_STR(pstr.str());
}
EXIT_BLOCK();
//CheckGhostSharedCount(__FILE__,__LINE__,ESET);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment