Commit 82e85bc2 authored by Kirill Terekhov's avatar Kirill Terekhov
Browse files

Commit for #2

Works on windows with parmetis-4.0.3. Added feature:  INMOST can provide
memory usage per element to ParMETIS.
parent f68a56a2
...@@ -107,6 +107,7 @@ if(USE_PARTITIONER_PARMETIS) ...@@ -107,6 +107,7 @@ if(USE_PARTITIONER_PARMETIS)
message("PARMETIS NOT FOUND") message("PARMETIS NOT FOUND")
else() else()
include_directories(${PARMETIS_INCLUDE_DIR}) include_directories(${PARMETIS_INCLUDE_DIR})
include_directories(${METIS_INCLUDE_DIR})
set(USE_PARTITIONER_PARMETIS ON) set(USE_PARTITIONER_PARMETIS ON)
message("PARMETIS FOUND") message("PARMETIS FOUND")
endif() endif()
......
...@@ -1233,8 +1233,10 @@ namespace INMOST ...@@ -1233,8 +1233,10 @@ namespace INMOST
__INLINE void * MGetLink (HandleType h, const Tag & t) {if( !t.isSparseByDim(GetHandleElementNum(h)) ) return MGetDenseLink(h,t); else {void * & q = MGetSparseLink(h,t); if( q == NULL ) q = calloc(1,t.GetRecordSize()); return q;}} __INLINE void * MGetLink (HandleType h, const Tag & t) {if( !t.isSparseByDim(GetHandleElementNum(h)) ) return MGetDenseLink(h,t); else {void * & q = MGetSparseLink(h,t); if( q == NULL ) q = calloc(1,t.GetRecordSize()); return q;}}
public: public:
/// For debug purposes /// For debug purposes
integer RetriveDataPositionForHandle(HandleType h) {return links[GetHandleElementNum(h)][GetHandleID(h)];} integer HandleDataPos (HandleType h) {return links[GetHandleElementNum(h)][GetHandleID(h)];}
/// For parmetis
/// return total number in bytes of occupied memory by element and its data
enumerator MemoryUsage (HandleType h);
Mesh(); Mesh();
Mesh(const Mesh & other); Mesh(const Mesh & other);
Mesh & operator =(Mesh const & other); Mesh & operator =(Mesh const & other);
......
...@@ -126,6 +126,37 @@ namespace INMOST ...@@ -126,6 +126,37 @@ namespace INMOST
allocated_meshes.push_back(this); allocated_meshes.push_back(this);
#endif #endif
} }
Storage::enumerator Mesh::MemoryUsage(HandleType h)
{
if( isValidHandle(h) )
{
integer etypenum = GetHandleElementNum(h);
enumerator ret = 2*sizeof(integer); //link and address occupied
for(Mesh::iteratorTag t = BeginTag(); t != EndTag(); ++t)
{
if( t->isDefinedByDim(etypenum) )
{
bool have_data = true;
if( t->isSparseByDim(etypenum) )
{
have_data = HaveData(h,*t);
if( have_data )
ret += sizeof(sparse_sub_record); //size occupied for storage of data link
}
if( have_data )
{
ret += t->GetRecordSize(); //all utilized data for fixed data, size of support structure for variable data
if( t->GetSize() == ENUMUNDEF )
ret += GetDataSize(h,*t)*t->GetBytesSize(); //actually occupied size for sparse data
}
}
if( !sparse_data[etypenum].empty() ) ret += sizeof(sparse_sub_type); //size needed to support sparse data
}
//Any additional impact of supporting huge structures over all elements may be added later
}
else return 0;
}
Mesh::Mesh(const Mesh & other) Mesh::Mesh(const Mesh & other)
:TagManager(other),Storage(NULL,ComposeHandle(MESH,0)) :TagManager(other),Storage(NULL,ComposeHandle(MESH,0))
......
...@@ -24,10 +24,10 @@ ...@@ -24,10 +24,10 @@
#define EXIT_FUNC() m->WriteTab(m->GetStream()) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; m->Exit(); m->WriteTab(m->GetStream()) << "</FUNCTION>" << std::endl; #define EXIT_FUNC() m->WriteTab(m->GetStream()) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; m->Exit(); m->WriteTab(m->GetStream()) << "</FUNCTION>" << std::endl;
#else #else
#define REPORT_MPI(x) x #define REPORT_MPI(x) x
#define REPORT_STR(x) #define REPORT_STR(x) {}
#define REPORT_VAL(str,x) #define REPORT_VAL(str,x) {}
#define ENTER_FUNC() #define ENTER_FUNC() {}
#define EXIT_FUNC() #define EXIT_FUNC() {}
#endif #endif
namespace INMOST namespace INMOST
...@@ -494,7 +494,7 @@ namespace INMOST ...@@ -494,7 +494,7 @@ namespace INMOST
if( m->GetMeshState() != Mesh::Parallel ) if( m->GetMeshState() != Mesh::Parallel )
m->ResolveShared(); m->ResolveShared();
if( m->Integer(m->LayersTag()) == 0 ) if( m->Integer(m->GetHandle(),m->LayersTag()) == 0 )
m->ExchangeGhost(1,FACE); m->ExchangeGhost(1,FACE);
//~ if( !m->GlobalIDTag().isValid() || !m->GlobalIDTag().isDefined(CELL) ) //~ if( !m->GlobalIDTag().isValid() || !m->GlobalIDTag().isDefined(CELL) )
m->AssignGlobalID(CELL); m->AssignGlobalID(CELL);
...@@ -546,6 +546,11 @@ namespace INMOST ...@@ -546,6 +546,11 @@ namespace INMOST
if( pa == Repartition ) if( pa == Repartition )
{ {
vsize.resize(mysize,1); vsize.resize(mysize,1);
int k = 0;
for(Mesh::iteratorCell it = m->BeginCell(); it != m->EndCell(); ++it) if( it->GetStatus() != Element::Ghost )
{
vsize[k++] = m->MemoryUsage(*it);
}
} }
...@@ -565,11 +570,12 @@ namespace INMOST ...@@ -565,11 +570,12 @@ namespace INMOST
if( have_adjwgt) adjwgt.reserve(mysize*ncon*nreserve); if( have_adjwgt) adjwgt.reserve(mysize*ncon*nreserve);
if( !have_adjwgt && !have_vwgt ) ncon = 1; if( !have_adjwgt && !have_vwgt ) ncon = 1;
//set constraint tolerances //set constraint tolerances
ubvec.resize(ncon,1.05); const real_t v = static_cast<real_t>(1.05);
ubvec.resize(ncon,v);
if( have_mwgt ) if( have_mwgt )
{ {
std::vector<real_t> l_tpwgts(ncon); std::vector<real_t> l_tpwgts(ncon);
for(idx_t q = 0; q < ncon; q++) l_tpwgts[q] = m->RealArray(GetWeight())[q]; for(idx_t q = 0; q < ncon; q++) l_tpwgts[q] = static_cast<real_t>(m->RealArray(m->GetHandle(),GetWeight())[q]);
REPORT_MPI(MPI_Allgather(&l_tpwgts[0],ncon,IDX_T,&tpwgts[0],ncon,IDX_T,m->GetCommunicator())); REPORT_MPI(MPI_Allgather(&l_tpwgts[0],ncon,IDX_T,&tpwgts[0],ncon,IDX_T,m->GetCommunicator()));
for(idx_t j = 0; j < ncon; j++) for(idx_t j = 0; j < ncon; j++)
...@@ -581,7 +587,7 @@ namespace INMOST ...@@ -581,7 +587,7 @@ namespace INMOST
} }
else else
{ {
double c = 1.0/(double)nparts; real_t c = static_cast<real_t>(1.0/static_cast<real_t>(nparts));
tpwgts.resize(nparts*ncon,c); tpwgts.resize(nparts*ncon,c);
} }
} }
...@@ -611,25 +617,29 @@ namespace INMOST ...@@ -611,25 +617,29 @@ namespace INMOST
//make graph //make graph
int k = 0; int k = 0;
adjacent<Face> faces,cells;
for(Mesh::iteratorCell it = m->BeginCell(); it != m->EndCell(); ++it) for(Mesh::iteratorCell it = m->BeginCell(); it != m->EndCell(); ++it)
if( it->GetStatus() != Element::Ghost ) if( it->GetStatus() != Element::Ghost )
{ {
idx_t sum = 0; idx_t sum = 0;
if( pa == Partition ) if( pa == Partition )
it->Centroid(&xyz[k*dim]);
{ {
faces = it->getFaces(); Storage::real xyz_temp[3] = {0.0,0.0,0.0};
for(adjacent<Face>::iterator jt = faces.begin(); jt != faces.end(); ++jt) it->Centroid(xyz_temp);
for(int q = 0; q < m->GetDimensions(); ++q)
xyz[k*dim+q] = static_cast<real_t>(xyz_temp[q]);
}
{
ElementArray<Face> faces = it->getFaces();
for(ElementArray<Face>::iterator jt = faces.begin(); jt != faces.end(); ++jt)
{ {
Cell * n = it->Neighbour(&*jt); Cell n = it->Neighbour(jt->self());
if( n != NULL ) if( n.isValid() )
{ {
adjncy.push_back(n->GlobalID()); adjncy.push_back(n->GlobalID());
if( have_adjwgt ) if( have_adjwgt )
{ {
for(idx_t q = 0; q < ncon; q++) for(idx_t q = 0; q < ncon; q++)
adjwgt.push_back(jt->RealArrayDF(GetWeight())[q]); adjwgt.push_back(jt->RealArray(GetWeight())[q]); //why adjwgt is not real???
} }
sum++; sum++;
} }
...@@ -641,7 +651,7 @@ namespace INMOST ...@@ -641,7 +651,7 @@ namespace INMOST
if( have_vwgt ) if( have_vwgt )
{ {
for(idx_t q = 0; q < ncon; q++) for(idx_t q = 0; q < ncon; q++)
vwgt[k*ncon+q] = it->RealArrayDF(GetWeight())[q]; vwgt[k*ncon+q] = it->RealArrayDF(GetWeight())[q]; //why vwgt is not real???
} }
} }
k++; k++;
...@@ -889,8 +899,30 @@ namespace INMOST ...@@ -889,8 +899,30 @@ namespace INMOST
MPI_Comm comm = m->GetCommunicator(); MPI_Comm comm = m->GetCommunicator();
idx_t * link_vtxdist = vtxdist.empty() ? NULL : &vtxdist[0];
idx_t * link_xadj = xadj.empty() ? NULL : &xadj[0];
idx_t * link_adjncy = adjncy.empty() ? NULL : &adjncy[0];
idx_t * link_vwgt = vwgt.empty() ? NULL : &vwgt[0];
idx_t * link_adjwgt = adjwgt.empty() ? NULL : &adjwgt[0];
real_t * link_tpwgts = tpwgts.empty() ? NULL : &tpwgts[0];
real_t * link_ubvec = ubvec.empty() ? NULL : &ubvec[0];
idx_t * link_options = options.empty() ? NULL : &options[0];
idx_t * link_part = part.empty() ? NULL : &part[0];
real_t * link_xyz = xyz.empty() ? NULL : &xyz[0];
idx_t * link_vsize = vsize.empty() ? NULL : &vsize[0];
REPORT_VAL("vtxdist",link_vtxdist);
REPORT_VAL("xadj",link_xadj);
REPORT_VAL("adjncy",link_adjncy);
REPORT_VAL("vwgt",link_vwgt);
REPORT_VAL("adjwgt",link_adjwgt);
REPORT_VAL("tpwgts",link_tpwgts);
REPORT_VAL("ubvec",link_ubvec);
REPORT_VAL("options",link_options);
REPORT_VAL("part",link_part);
REPORT_VAL("xyz",link_xyz);
REPORT_VAL("vsize",link_vsize);
switch(pa) switch(pa)
{ {
//~ case Partition: //~ case Partition:
...@@ -899,22 +931,25 @@ namespace INMOST ...@@ -899,22 +931,25 @@ namespace INMOST
//~ &ubvec[0],&options[0],&edgecut,&part[0],&comm); //~ &ubvec[0],&options[0],&edgecut,&part[0],&comm);
//~ break; //~ break;
case Refine: case Refine:
result = ParMETIS_V3_RefineKway(&vtxdist[0],&xadj[0],&adjncy[0],&vwgt[0],&adjwgt[0], REPORT_STR("run refine");
&wgtflag,&numflag,&ncon,&nparts,&tpwgts[0], result = ParMETIS_V3_RefineKway(link_vtxdist,link_xadj,link_adjncy,link_vwgt,link_adjwgt,
&ubvec[0],&options[0],&edgecut,&part[0],&comm); &wgtflag,&numflag,&ncon,&nparts,link_tpwgts,
link_ubvec,link_options,&edgecut,link_part,&comm);
break; break;
case Partition: case Partition:
result = ParMETIS_V3_PartGeomKway(&vtxdist[0],&xadj[0],&adjncy[0],&vwgt[0],&adjwgt[0], REPORT_STR("run partition");
&wgtflag,&numflag,&ndims,&xyz[0],&ncon,&nparts,&tpwgts[0], result = ParMETIS_V3_PartGeomKway(link_vtxdist,link_xadj,link_adjncy,link_vwgt,link_adjwgt,
&ubvec[0],&options[0],&edgecut,&part[0],&comm); &wgtflag,&numflag,&ndims,link_xyz,&ncon,&nparts,link_tpwgts,
link_ubvec,link_options,&edgecut,link_part,&comm);
break; break;
//~ case P_PartGeom: //~ case P_PartGeom:
//~ result = ParMETIS_V3_PartGeom(&vtxdist[0],&ndims,&xyz[0],&part[0],&comm); //~ result = ParMETIS_V3_PartGeom(&vtxdist[0],&ndims,&xyz[0],&part[0],&comm);
//~ break; //~ break;
case Repartition: case Repartition:
result = ParMETIS_V3_AdaptiveRepart(&vtxdist[0],&xadj[0],&adjncy[0],&vwgt[0],&vsize[0],&adjwgt[0], REPORT_STR("run repartition");
&wgtflag,&numflag,&ncon,&nparts,&tpwgts[0], result = ParMETIS_V3_AdaptiveRepart(link_vtxdist,link_xadj,link_adjncy,link_vwgt,link_vsize,
&ubvec[0],&itr,&options[0],&edgecut,&part[0],&comm); link_adjwgt,&wgtflag,&numflag,&ncon,&nparts,link_tpwgts,
link_ubvec,&itr,link_options,&edgecut,link_part,&comm);
break; break;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment