138 mEnv->debug(DETAILED_STATUS, std::string(
"Entering AlgBlock"));
140 int myrank = mEnv->myRank_;
141 int nprocs = mEnv->numProcs_;
147 int procDim = sqrt(nprocs);
148 assert(procDim * procDim == nprocs);
150 int myProcRow = myrank / procDim;
151 int myProcCol = myrank % procDim;
161 Teuchos::ParameterList params1D(
"Params for 1D partitioning");
163 params1D.set(
"algorithm",
"block");
164 params1D.set(
"imbalance_tolerance", 1.1);
165 params1D.set(
"num_global_parts", procDim);
172 MatrixAdapter<user_t, userCoord_t> *adapterPtr =
173 const_cast<MatrixAdapter<user_t, userCoord_t>*
>(mMatrixAdapter.getRawPtr());
196 const part_t *parts = solution1D.getPartListView();
215 size_t nLocIDs = mMatrixAdapter->getLocalNumRows();
217 std::vector<std::vector<gno_t> > idsInProcColI(procDim);
218 Teuchos::ArrayRCP<gno_t> colIDs;
220 for(
size_t i=0; i<nLocIDs; i++)
224 idsInProcColI[ parts[i] ].push_back(rowIds[i]);
236 for(
int procColI=0; procColI<procDim; procColI++)
242 if(myProcCol==procColI && myProcRow==0)
244 assert(myrank==procColI);
247 std::set<gno_t> gidSet;
252 for(
int i=0;i<idsInProcColI[procColI].size();i++)
254 gidSet.insert(idsInProcColI[procColI][i]);
261 std::vector<gno_t> recvBuf;
262 for(
int src=0;src<nprocs;src++)
268 Teuchos::receive<int,int>(*mProblemComm, src, 1, &buffSize);
272 recvBuf.resize(buffSize);
274 Teuchos::receive<int,gno_t>(*mProblemComm, src, buffSize, recvBuf.data());
276 for(
int i=0;i<buffSize;i++)
278 gidSet.insert(recvBuf[i]);
288 colIDs.resize(gidSet.size());
290 typename std::set<gno_t>::const_iterator iter;
292 for (iter=gidSet.begin(); iter!=gidSet.end(); ++iter)
294 colIDs[indx] = *iter;
306 int sizeToSend = idsInProcColI[procColI].size();
311 Teuchos::send<int,int>(*mProblemComm,1,&sizeToSend,procColI);
313 Teuchos::send<int,gno_t>(*mProblemComm,sizeToSend,idsInProcColI[procColI].data(),procColI);
335 for(
int procColRank=0; procColRank<procDim; procColRank++)
338 int dstRank = procColRank*procDim + myProcCol;
342 int sizeToSend = colIDs.size();
344 Teuchos::send<int,int>(*mProblemComm,1,&sizeToSend,dstRank);
345 Teuchos::send<int,gno_t>(*mProblemComm,sizeToSend,colIDs.get(),dstRank);
360 int srcRank = myProcCol;
363 Teuchos::receive<int,int>(*mProblemComm, srcRank, 1, &buffSize);
365 colIDs.resize(buffSize);
366 Teuchos::receive<int,gno_t>(*mProblemComm, srcRank, buffSize, colIDs.get());
380 ArrayRCP<gno_t> domRangeIDs;
382 size_t nCols = colIDs.size();
383 lno_t nColsDivDim = nCols / procDim;
384 lno_t nColsModDim = nCols % procDim;
390 if(myProcRow < nColsModDim)
392 sColIndx = myProcRow * (nColsDivDim+1);
393 domRangeSize = nColsDivDim+1;
398 sColIndx = nColsModDim*(nColsDivDim+1) + (myProcRow-nColsModDim) * nColsDivDim;
399 domRangeSize = nColsDivDim;
402 domRangeIDs.resize(domRangeSize);
404 for(
size_t i=0;i<domRangeSize;i++)
406 domRangeIDs[i] = colIDs[sColIndx+i];
421 std::vector<int> subcommRanks(procDim);
423 for(
unsigned int i=0; i<procDim; i++)
425 subcommRanks[i] = myProcRow*procDim + i;
428 ArrayView<const int> subcommRanksView = Teuchos::arrayViewFromVector (subcommRanks);
430 RCP<Teuchos::Comm<int> > rowcomm = mProblemComm->createSubcommunicator(subcommRanksView);
436 size_t maxProcRowNCols=0;
438 Teuchos::reduceAll<int, size_t>(*rowcomm,Teuchos::REDUCE_MAX,1,&domRangeSize,&maxProcRowNCols);
444 gno_t MAXVAL = std::numeric_limits<gno_t>::max();
446 std::vector<gno_t> locRowIDs(maxProcRowNCols,MAXVAL);
448 Teuchos::ArrayRCP<gno_t> rowIDs(maxProcRowNCols * procDim);
452 for(
size_t i=0;i<domRangeIDs.size();i++)
454 locRowIDs[i] = domRangeIDs[i];
458 Teuchos::gatherAll<int,gno_t>(*rowcomm,maxProcRowNCols, locRowIDs.data(),
459 maxProcRowNCols*procDim, rowIDs.get());
463 std::vector<int>().swap(locRowIDs);
469 std::set<gno_t> setRowIDs;
471 for(
size_t i=0;i<rowIDs.size();i++)
473 if(rowIDs[i]!=MAXVAL)
475 setRowIDs.insert(rowIDs[i]);
483 rowIDs.resize(setRowIDs.size());
485 typename std::set<gno_t>::const_iterator iter;
488 for(iter=setRowIDs.begin(); iter!=setRowIDs.end(); ++iter)
490 rowIDs[indx] = *iter;
500 solution->setIDLists(rowIDs,colIDs,domRangeIDs,domRangeIDs);
503 mEnv->debug(DETAILED_STATUS, std::string(
"Exiting AlgMatrix"));