Commit 7a5161c1 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

Sync fixes (there is still a bug and optimizations to be done)

parent 92987ad4
...@@ -388,7 +388,7 @@ namespace INMOST ...@@ -388,7 +388,7 @@ namespace INMOST
//create a tag that stores maximal refinement level of each element //create a tag that stores maximal refinement level of each element
level = m->CreateTag("REFINEMENT_LEVEL",DATA_INTEGER,CELL|FACE|EDGE|NODE|ESET,NONE,1); level = m->CreateTag("REFINEMENT_LEVEL",DATA_INTEGER,CELL|FACE|EDGE|NODE|ESET,NONE,1);
//tag_status = m->CreateTag("TAG_STATUS",DATA_INTEGER,CELL|FACE|EDGE|NODE,NONE,1); //tag_status = m->CreateTag("TAG_STATUS",DATA_INTEGER,CELL|FACE|EDGE|NODE,NONE,1);
set_id = m->CreateTag("SET_ID",DATA_INTEGER,CELL,NONE,1); set_id = m->CreateTag("SET_ID",DATA_INTEGER,CELL|ESET,ESET,1);
//tag_an = m->CreateTag("TAG_AN",DATA_INTEGER,CELL|FACE|EDGE|NODE,NONE,1); //tag_an = m->CreateTag("TAG_AN",DATA_INTEGER,CELL|FACE|EDGE|NODE,NONE,1);
//ref_tag = m->CreateTag("REF",DATA_REFERENCE,CELL|FACE|EDGE|NODE,NONE); //ref_tag = m->CreateTag("REF",DATA_REFERENCE,CELL|FACE|EDGE|NODE,NONE);
//create a tag that stores links to all the hanging nodes of the cell //create a tag that stores links to all the hanging nodes of the cell
...@@ -398,6 +398,28 @@ namespace INMOST ...@@ -398,6 +398,28 @@ namespace INMOST
size = m->GetProcessorsNumber(); size = m->GetProcessorsNumber();
rank = m->GetProcessorRank(); rank = m->GetProcessorRank();
} }
void ReportSub(ElementSet root, int tab, std::fstream & fout)
{
if( root.HaveChild() )
{
for(ElementSet jt = root.GetChild(); jt.isValid(); jt = jt.GetSibling())
ReportSub(jt,tab+1,fout);
}
Storage::integer_array parr = root.IntegerArray(root.GetMeshLink()->ProcessorsTag());
for(int q = 0; q < tab; ++q) fout << " ";
fout << "set " << root.GetName() << " procs ";
for(unsigned q = 0; q < parr.size(); ++q) fout << parr[q] << " ";
fout << " owner " << root.Integer(root.GetMeshLink()->OwnerTag());
fout << " kids " << root.Size();
fout << " status " << Element::StatusName(root.GetStatus()) << std::endl;
}
void AdaptiveMesh::ReportSets(std::fstream & fout)
{
for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it)
if( !it->HaveParent() ) ReportSub(it->self(),0,fout);
}
AdaptiveMesh::~AdaptiveMesh() AdaptiveMesh::~AdaptiveMesh()
{ {
...@@ -405,10 +427,11 @@ namespace INMOST ...@@ -405,10 +427,11 @@ namespace INMOST
//as extension of class mesh in limited code span //as extension of class mesh in limited code span
} }
void AdaptiveMesh::CheckParentSet(std::string file, int line, TagInteger indicator) void AdaptiveMesh::CheckParentSet(std::string file, int line)//, TagInteger indicator)
{ {
ENTER_FUNC(); ENTER_FUNC();
int err = 0; int err = 0;
Storage::integer_array procs;
for(Mesh::iteratorCell it = m->BeginCell(); it != m->EndCell(); ++it) for(Mesh::iteratorCell it = m->BeginCell(); it != m->EndCell(); ++it)
{ {
if( parent_set[*it] == InvalidHandle() ) if( parent_set[*it] == InvalidHandle() )
...@@ -423,21 +446,25 @@ namespace INMOST ...@@ -423,21 +446,25 @@ namespace INMOST
std::cout << m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl; std::cout << m->GetProcessorRank() << " parent set is something else " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl;
err++; err++;
} }
else if( parent_set[*it] != root.GetHandle() && (!indicator.isValid() || indicator[*it]) ) else if( parent_set[*it] != root.GetHandle() )//&& (!indicator.isValid() || indicator[*it]) )
{ {
ElementSet set(m,parent_set[*it]); ElementSet set(m,parent_set[*it]);
if( !set.HaveParent() ) if( !set.HaveParent() )
{ {
REPORT_STR(m->GetProcessorRank() << REPORT_STR(m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) << " name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) << " kids " << set.Size() <<
" does not have parent " << " does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it]); " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " owner " << it->Integer(m->OwnerTag()) << " lvl " << level[*it]);
std::cout << m->GetProcessorRank() << std::cout << m->GetProcessorRank() <<
" parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) << " parent set " << ElementTypeName(GetHandleElementType(parent_set[*it])) << ":" << GetHandleID(parent_set[*it]) <<
" name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) << " name " << set.GetName() << " owner " << set.Integer(m->OwnerTag()) << " status " << Element::StatusName(set.GetStatus()) << " kids " << set.Size() <<
" does not have parent " << " does not have parent " <<
" on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " " << level[*it] << std::endl; " on CELL:" << it->LocalID() << " " << Element::StatusName(it->GetStatus()) << " owner " << it->Integer(m->OwnerTag()) << " lvl " << level[*it] << std::endl;
procs = it->IntegerArray(m->ProcessorsTag());
std::cout << "cell procs:"; for(unsigned q = 0; q < procs.size(); ++q) std::cout << " " << procs[q]; std::cout << std::endl;
procs = set.IntegerArray(m->ProcessorsTag());
std::cout << "eset procs:"; for(unsigned q = 0; q < procs.size(); ++q) std::cout << " " << procs[q]; std::cout << std::endl;
err++; err++;
} }
else else
...@@ -492,6 +519,7 @@ namespace INMOST ...@@ -492,6 +519,7 @@ namespace INMOST
//EXIT_BLOCK(); //EXIT_BLOCK();
m->CheckSetLinks(__FILE__,__LINE__); m->CheckSetLinks(__FILE__,__LINE__);
CheckParentSet(__FILE__,__LINE__);
/* /*
ENTER_BLOCK(); ENTER_BLOCK();
m->ResolveSets(); m->ResolveSets();
...@@ -781,13 +809,11 @@ namespace INMOST ...@@ -781,13 +809,11 @@ namespace INMOST
// //
//} //}
//split the cell //split the cell
ElementArray<Cell> new_cells = Cell::SplitCell(c,internal_faces,0);
std::sort(new_cells.begin(),new_cells.end(),Mesh::CentroidComparator(m));
//retrive parent set //retrive parent set
ElementSet parent(m,parent_set[c]); ElementSet parent(m,parent_set[c]);
//create set corresponding to old coarse cell //create set corresponding to old coarse cell
Storage::real cnt[3]; //Storage::real cnt[3];
c.Centroid(cnt); //c.Centroid(cnt);
std::stringstream set_name; std::stringstream set_name;
//set_name << parent.GetName() << "_C" << c.GlobalID(); //rand may be unsafe //set_name << parent.GetName() << "_C" << c.GlobalID(); //rand may be unsafe
if( parent == root ) if( parent == root )
...@@ -802,16 +828,32 @@ namespace INMOST ...@@ -802,16 +828,32 @@ namespace INMOST
std::cout << rank << " set " << set_name.str() << " for cell " << c.GlobalID() << " " << Element::StatusName(c.GetStatus()) << " already exists" << std::endl; std::cout << rank << " set " << set_name.str() << " for cell " << c.GlobalID() << " " << Element::StatusName(c.GetStatus()) << " already exists" << std::endl;
if( check_set->HaveParent() ) if( check_set->HaveParent() )
std::cout << rank << " parent is " << check_set->GetParent()->GetName() << " cell parent is " << parent.GetName() << std::endl; std::cout << rank << " parent is " << check_set->GetParent()->GetName() << " cell parent is " << parent.GetName() << std::endl;
std::cout << rank << " Elements: "; std::cout << rank << " Elements of " << check_set.GetName() << ": ";
for(ElementSet::iterator it = check_set.Begin(); it != check_set.End(); ++it) for(ElementSet::iterator it = check_set.Begin(); it != check_set.End(); ++it)
std::cout << ElementTypeName(it->GetElementType()) << ":" << it->LocalID() << "," << it->GlobalID() << "," << Element::StatusName(c.GetStatus()) << "," << level[*it] << " "; std::cout << ElementTypeName(it->GetElementType()) << ":" << it->LocalID() << "," << it->GlobalID() << "," << Element::StatusName(c.GetStatus()) << "," << level[*it] << "," << indicator[*it] << " ";
std::cout << std::endl; std::cout << std::endl;
std::cout << rank << " Elements of " << parent.GetName() << ": ";
for(ElementSet::iterator it = parent.Begin(); it != parent.End(); ++it)
std::cout << ElementTypeName(it->GetElementType()) << ":" << it->LocalID() << "," << it->GlobalID() << "," << Element::StatusName(c.GetStatus()) << "," << level[*it] << "," << indicator[*it] << " ";
std::cout << std::endl;
if( parent.HaveChild() )
{
std::cout << rank << " Children of " << parent.GetName() << ": ";
for(ElementSet jt = parent.GetChild(); jt.isValid(); jt = jt.GetSibling() )
std::cout << jt.GetName() << " size " << jt.Size() << " ";
std::cout << std::endl;
}
exit(-1); exit(-1);
} }
ElementSet cell_set = m->CreateSetUnique(set_name.str()).first; ElementSet cell_set = m->CreateSetUnique(set_name.str()).first;
//cell_set->SetExchange(ElementSet::SYNC_ELEMENTS_ALL); //cell_set->SetExchange(ElementSet::SYNC_ELEMENTS_ALL);
level[cell_set] = level[c]+1; level[cell_set] = level[c]+1;
set_id[cell_set] = set_id[c];
ElementArray<Cell> new_cells = Cell::SplitCell(c,internal_faces,0);
std::sort(new_cells.begin(),new_cells.end(),Mesh::CentroidComparator(m));
//set up increased level for the new cells //set up increased level for the new cells
for(ElementArray<Cell>::size_type kt = 0; kt < new_cells.size(); ++kt) for(ElementArray<Cell>::size_type kt = 0; kt < new_cells.size(); ++kt)
{ {
...@@ -867,6 +909,31 @@ namespace INMOST ...@@ -867,6 +909,31 @@ namespace INMOST
EXIT_BLOCK(); EXIT_BLOCK();
m->ExchangeData(parent_set,CELL,0); m->ExchangeData(parent_set,CELL,0);
/*
ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( set.GetName().substr(0,3) == "AM_" )
{
//int imax = -1, imin = INT_MAX;
//for(ElementSet::iterator jt = set.Begin(); jt != set.End(); ++jt)
//{
// imax = std::max(imax,indicator[*jt]);
// imin = std::min(imin,indicator[*jt]);
//}
//std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax;
//if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName();
//std::cout << std::endl;
//if( !set.HaveChild() )
set.SynchronizeSetParents();
}
}
m->ExchangeMarked();
EXIT_BLOCK();
*/
CheckParentSet(__FILE__,__LINE__);
//m->Save("after_refine"+std::to_string(fi)+".pvtk"); //m->Save("after_refine"+std::to_string(fi)+".pvtk");
//std::cout << "Save after_refine"+std::to_string(fi)+".pvtk" << std::endl; //std::cout << "Save after_refine"+std::to_string(fi)+".pvtk" << std::endl;
...@@ -976,7 +1043,7 @@ namespace INMOST ...@@ -976,7 +1043,7 @@ namespace INMOST
//SynchronizeIndicated(indicator); //SynchronizeIndicated(indicator);
//EXIT_BLOCK(); //EXIT_BLOCK();
CheckParentSet(__FILE__,__LINE__);
int schedule_counter = 1; //indicates order in which refinement will be scheduled int schedule_counter = 1; //indicates order in which refinement will be scheduled
int scheduled = 1, unscheduled = 0; //indicates that at least one element was scheduled on current sweep int scheduled = 1, unscheduled = 0; //indicates that at least one element was scheduled on current sweep
...@@ -1259,23 +1326,9 @@ namespace INMOST ...@@ -1259,23 +1326,9 @@ namespace INMOST
m->ExchangeData(parent_set,CELL,0); m->ExchangeData(parent_set,CELL,0);
m->ResolveSets();
//CheckParentSet();
/* /*
ENTER_BLOCK();
//m->CheckCentroids(__FILE__,__LINE__);
m->ExchangeData(parent_set,CELL,0);
//m->CheckCentroids(__FILE__,__LINE__);
m->ResolveSets();
//m->CheckCentroids(__FILE__,__LINE__);
//m->ExchangeData(parent_set,CELL,0);
//m->CheckCentroids(__FILE__,__LINE__);
m->ExchangeData(hanging_nodes,CELL | FACE,0);
//m->CheckCentroids(__FILE__,__LINE__);
EXIT_BLOCK();
*/
//m->Barrier();
ENTER_BLOCK(); ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) ) for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
...@@ -1292,17 +1345,19 @@ namespace INMOST ...@@ -1292,17 +1345,19 @@ namespace INMOST
//std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax; //std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax;
//if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName(); //if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName();
//std::cout << std::endl; //std::cout << std::endl;
if( indicator[set] != 0 ) //if( indicator[set] != 0 )
set.SynchronizeSetParents(); //if( !set.HaveChild() )
set.SynchronizeSetParents();
} }
} }
EXIT_BLOCK(); EXIT_BLOCK();
*/
//m->Barrier(); //m->Barrier();
//std::cout << m->GetProcessorRank() << " call exchange marked" << std::endl; //std::cout << m->GetProcessorRank() << " call exchange marked" << std::endl;
m->ExchangeMarked(); m->ExchangeMarked();
//std::cout << m->GetProcessorRank() << " finish exchange marked" << std::endl; //std::cout << m->GetProcessorRank() << " finish exchange marked" << std::endl;
//m->Barrier(); //m->Barrier();
CheckParentSet(__FILE__,__LINE__,indicator); CheckParentSet(__FILE__,__LINE__);//,indicator);
//std::fstream fout("sets"+std::to_string(m->GetProcessorRank())+".txt",std::ios::out); //std::fstream fout("sets"+std::to_string(m->GetProcessorRank())+".txt",std::ios::out);
//for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it) //for(Mesh::iteratorSet it = m->BeginSet(); it != m->EndSet(); ++it)
...@@ -1317,7 +1372,7 @@ namespace INMOST ...@@ -1317,7 +1372,7 @@ namespace INMOST
m->BeginModification(); m->BeginModification();
while(schedule_counter) while(schedule_counter)
{ {
CheckParentSet(__FILE__,__LINE__,indicator); //CheckParentSet(__FILE__,__LINE__);//,indicator);
//CheckParentSet(); //CheckParentSet();
//fout << "schedule_counter " << schedule_counter << std::endl; //fout << "schedule_counter " << schedule_counter << std::endl;
//unite cells //unite cells
...@@ -1392,6 +1447,7 @@ namespace INMOST ...@@ -1392,6 +1447,7 @@ namespace INMOST
assert(center_node.size() == 1); assert(center_node.size() == 1);
ElementArray<Node> hanging = center_node[0].BridgeAdjacencies2Node(EDGE); ElementArray<Node> hanging = center_node[0].BridgeAdjacencies2Node(EDGE);
Cell v = Cell::UniteCells(unite_cells,0); Cell v = Cell::UniteCells(unite_cells,0);
set_id[v] = set_id[parent];
//connect hanging nodes to the cell //connect hanging nodes to the cell
assert(hanging_nodes[v].size() == 0); assert(hanging_nodes[v].size() == 0);
for(ElementArray<Node>::size_type kt = 0; kt < hanging.size(); ++kt) for(ElementArray<Node>::size_type kt = 0; kt < hanging.size(); ++kt)
...@@ -1541,6 +1597,31 @@ namespace INMOST ...@@ -1541,6 +1597,31 @@ namespace INMOST
//fout.close(); //fout.close();
m->ExchangeData(parent_set,CELL,0); m->ExchangeData(parent_set,CELL,0);
/*
ENTER_BLOCK();
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( set.GetName().substr(0,3) == "AM_" )
{
//int imax = -1, imin = INT_MAX;
//for(ElementSet::iterator jt = set.Begin(); jt != set.End(); ++jt)
//{
// imax = std::max(imax,indicator[*jt]);
// imin = std::min(imin,indicator[*jt]);
//}
//std::cout << "on proc " << m->GetProcessorRank() << " set " << set.GetName() << " size " << set.Size() << " set indicator " << indicator[set] << " elements indicator " << imin << ":" << imax;
//if( set.HaveParent() ) std::cout << " parent " << set.GetParent().GetName();
//std::cout << std::endl;
//if( !set.HaveChild() )
set.SynchronizeSetParents();
}
}
m->ExchangeMarked();
EXIT_BLOCK();
*/
CheckParentSet(__FILE__,__LINE__);
//m->Save("after_coarse"+std::to_string(fi)+".pvtk"); //m->Save("after_coarse"+std::to_string(fi)+".pvtk");
//std::cout << "Save after_coarse"+std::to_string(fi)+".pvtk" << std::endl; //std::cout << "Save after_coarse"+std::to_string(fi)+".pvtk" << std::endl;
...@@ -1600,105 +1681,4 @@ namespace INMOST ...@@ -1600,105 +1681,4 @@ namespace INMOST
return ret != 0; return ret != 0;
} }
#if defined(USE_PARTITIONER)
void AdaptiveMesh::SetNewOwner(ElementSet set, TagInteger owner)
{
if( set.HaveChild() )
{
if( set.Empty() ) owner[set] = INT_MAX;
for(ElementSet chld = set.GetChild(); chld.isValid(); chld = chld.GetSibling())
{
SetNewOwner(chld,owner);
owner[set] = std::min(owner[set],owner[chld]);
}
}
//for(ElementSet::iterator it = set.Begin(); it != set.End(); ++it)
// owner[set] = std::min(owner[set],owner[*it]);
}
void AdaptiveMesh::SetNewProcs(ElementSet set, TagIntegerArray procs)
{
std::vector<int> tmp;
for(ElementSet chld = set.GetChild(); chld.isValid(); chld = chld.GetSibling())
{
SetNewProcs(chld,procs);
Storage::integer_array pm = procs[set];
Storage::integer_array pc = procs[chld];
tmp.resize(pm.size()+pc.size());
tmp.resize(std::set_union(pm.begin(),pm.end(),pc.begin(),pc.end(),tmp.begin())-tmp.begin());
pm.replace(pm.begin(),pm.end(),tmp.begin(),tmp.end());
}
}
void AdaptiveMesh::RestoreParent(ElementSet set)
{
for(ElementSet chld = set.GetChild(); chld.isValid(); chld = chld.GetSibling())
RestoreParent(chld);
for(ElementSet::iterator it = set.Begin(); it != set.End(); ++it)
parent_set[*it] = set.GetHandle();
}
void AdaptiveMesh::Repartition()
{
if( !root.isValid() ) return;
TagInteger redist = m->RedistributeTag();
TagInteger new_owner = m->CreateTag("TEMPORARY_NEW_OWNER",DATA_INTEGER,ESET, NONE,1); //workaround for sets
TagIntegerArray new_procs = m->CreateTag("TEMPORARY_NEW_PROCESSORS",DATA_INTEGER,ESET, NONE);
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
new_owner[set] = INT_MAX;
new_procs[set].clear();
}
std::cout << __FUNCTION__ << ":" << __FILE__ << ":" << __LINE__ << std::endl;
for(Storage::integer it = 0; it < m->CellLastLocalID(); ++it) if( m->isValidCell(it) )
{
Cell c = m->CellByLocalID(it);
ElementSet parent(m,parent_set[c]);
std::cout << "cell:" << it << " gid " << c.GlobalID() << " has parent " << parent.LocalID() << " " << parent.GetName() << " redist " << redist[c] << std::endl;
new_owner[parent] = std::min(new_owner[parent],redist[c]);
Storage::integer_array procs = new_procs[parent];
Storage::integer_array::iterator insp = std::lower_bound(procs.begin(),procs.end(),redist[c]);
if( insp == procs.end() || *insp != redist[c] ) procs.insert(insp,redist[c]);
}
SetNewOwner(root,new_owner);
SetNewProcs(root,new_procs);
m->ReduceData(new_owner,ESET,0,ReduceMin);
m->ExchangeData(new_owner,ESET,0);
m->ReduceData(new_procs,ESET,0,ReduceUnion);
m->ExchangeData(new_procs,ESET,0);
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
Storage::integer_array old_p = set->IntegerArray(m->ProcessorsTag());
Storage::integer_array new_p = new_procs[set];
Storage::integer_array sendto = set->IntegerArray(m->SendtoTag());
sendto.resize(std::max(old_p.size(),new_p.size()));
sendto.resize(std::set_difference(new_p.begin(),new_p.end(),old_p.begin(),old_p.end(),sendto.begin())-sendto.begin());
}
std::cout << __FUNCTION__ << ":" << __FILE__ << ":" << __LINE__ << std::endl;
for(Storage::integer it = 0; it < m->EsetLastLocalID(); ++it) if( m->isValidElementSet(it) )
{
ElementSet set = m->EsetByLocalID(it);
if( new_owner[set] == INT_MAX ) std::cout << "no new owner for set " << it << " " << set.GetName() << std::endl;
}
for(Storage::integer it = 0; it < m->CellLastLocalID(); ++it) if( m->isValidCell(it) )
{
Cell c = m->CellByLocalID(it);
if( parent_set[c] == InvalidHandle() ) std::cout << "No parent set for cell:" << it << " gid " << c.GlobalID() << std::endl;
}
m->BeginModification();
m->Redistribute();
m->ExchangeData(hanging_nodes,CELL|FACE,0); //maybe will work
m->ApplyModification(); //this will correctly remove links
m->EndModification();
m->ReorderEmpty(CELL|FACE|EDGE|NODE);
RestoreParent(root);
std::cout << __FUNCTION__ << ":" << __FILE__ << ":" << __LINE__ << std::endl;
for(Storage::integer it = 0; it < m->CellLastLocalID(); ++it) if( m->isValidCell(it) )
{
Cell c = m->CellByLocalID(it);
if( parent_set[c] == InvalidHandle() ) std::cout << "No parent set for cell:" << it << " gid " << c.GlobalID() << std::endl;
}
}
#endif
} }
...@@ -16,14 +16,12 @@ namespace INMOST ...@@ -16,14 +16,12 @@ namespace INMOST
int size; int size;
/// Prepare sets for coarsements. /// Prepare sets for coarsements.
/// Do not do this in constructor, since mesh may contain no cells. /// Do not do this in constructor, since mesh may contain no cells.
void CheckParentSet(std::string file, int line, TagInteger indicator);
void PrepareSet(); void PrepareSet();
//void PrintSetLocal(std::string offset, ElementSet it, std::stringstream& ss); //void PrintSetLocal(std::string offset, ElementSet it, std::stringstream& ss);
//void SynchronizeIndicated(TagInteger& indicator); //void SynchronizeIndicated(TagInteger& indicator);
void SetNewOwner(ElementSet set, TagInteger owner);
void SetNewProcs(ElementSet set, TagIntegerArray procs);
void RestoreParent(ElementSet set);
public: public:
void ReportSets(std::fstream & fout);
void CheckParentSet(std::string file, int line);//, TagInteger indicator);
TagReference parent_set; //<Link to the set that contains an element. TagReference parent_set; //<Link to the set that contains an element.
TagReferenceArray hanging_nodes; //< Link to current hanging nodes of the cell. TagReferenceArray hanging_nodes; //< Link to current hanging nodes of the cell.
TagInteger level; //< Refinement level of the cell TagInteger level; //< Refinement level of the cell
...@@ -40,7 +38,6 @@ namespace INMOST ...@@ -40,7 +38,6 @@ namespace INMOST
void ClearData(); void ClearData();
void PrintSet(std::ostream & fout, ElementSet set); void PrintSet(std::ostream & fout, ElementSet set);
void SetModel(Model * mm) {model = mm;} void SetModel(Model * mm) {model = mm;}
void Repartition();
//void Test(); //void Test();
//void PrintMesh(std::ostream& os, int cell = 0, int face = 0, int edge = 0, int node = 0); //void PrintMesh(std::ostream& os, int cell = 0, int face = 0, int edge = 0, int node = 0);
//void PrintSet(); //void PrintSet();
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
using namespace INMOST; using namespace INMOST;
std::string file_format = ".pmf";
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {
Mesh::Initialize(&argc,&argv); Mesh::Initialize(&argc,&argv);
...@@ -22,12 +24,15 @@ int main(int argc, char ** argv) ...@@ -22,12 +24,15 @@ int main(int argc, char ** argv)
Partitioner p(&m); Partitioner p(&m);
if( true ) if( true )
{ {
m.Barrier();
std::cout << "before on " << m.GetProcessorRank() << " " << m.NumberOfCells() << std::endl; std::cout << "before on " << m.GetProcessorRank() << " " << m.NumberOfCells() << std::endl;
p.SetMethod(Partitioner::INNER_KMEANS,Partitioner::Partition); p.SetMethod(Partitioner::INNER_KMEANS,Partitioner::Partition);
//p.SetMethod(Partitioner::Parmetis,Partitioner::Refine);
p.Evaluate(); p.Evaluate();
m.Redistribute(); m.Redistribute();
m.ReorderEmpty(CELL|FACE|EDGE|NODE); m.ReorderEmpty(CELL|FACE|EDGE|NODE);
std::cout << "after on " << m.GetProcessorRank() << " " << m.NumberOfCells() << std::endl; std::cout << "after on " << m.GetProcessorRank() << " " << m.NumberOfCells() << std::endl;
p.SetMethod(Partitioner::Parmetis,Partitioner::Repartition);
} }
#endif #endif
m.ExchangeGhost(2,FACE); m.ExchangeGhost(2,FACE);
...@@ -49,7 +54,7 @@ int main(int argc, char ** argv) ...@@ -49,7 +54,7 @@ int main(int argc, char ** argv)
//bounding box around mesh //bounding box around mesh
Storage::real cmax[3] = {-1.0e20,-1.0e20,-1.0e20}; Storage::real cmax[3] = {-1.0e20,-1.0e20,-1.0e20};
Storage::real cmin[3] = {+1.0e20,+1.0e20,+1.0e20}; Storage::real cmin[3] = {+1.0e20,+1.0e20,+1.0e20};
Storage::real xyz[3], r, q, cnt[3]; Storage::real xyz[3], r, q, cnt[3], cnt0[3], r0;
//find bounding box around mesh //find bounding box around mesh
for(Mesh::iteratorNode it = m.BeginNode(); it != m.EndNode(); ++it) for(Mesh::iteratorNode it = m.BeginNode(); it != m.EndNode(); ++it)
{ {
...@@ -61,17 +66,23 @@ int main(int argc, char ** argv) ...@@ -61,17 +66,23 @@ int main(int argc, char ** argv)
} }
m.AggregateMax(cmax,3); m.AggregateMax(cmax,3);
m.AggregateMin(cmin,3); m.AggregateMin(cmin,3);
r = 1; r0 = 1;
for(int d = 0; d < 3; ++d) for(int d = 0; d < 3; ++d)
{ {
r *= cmax[d]-cmin[d]; r0 *= cmax[d]-cmin[d];
cnt[d] = (cmax[d]+cmin[d])*0.5; cnt[d] = (cmax[d]+cmin[d])*0.5;
cnt0[d] = cnt[d];
} }
r = pow(r,1.0/3.0)/20.0; //r = pow(r,1.0/3.0)/20.0;
r0 = pow(r0,1.0/3.0);