Zoltan2
Loading...
Searching...
No Matches
APFMeshAdapterTest.cpp
Go to the documentation of this file.
1// @HEADER
2//
3// ***********************************************************************
4//
5// Zoltan2: A package of combinatorial algorithms for scientific computing
6// Copyright 2012 Sandia Corporation
7//
8// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9// the U.S. Government retains certain rights in this software.
10//
11// Redistribution and use in source and binary forms, with or without
12// modification, are permitted provided that the following conditions are
13// met:
14//
15// 1. Redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer.
17//
18// 2. Redistributions in binary form must reproduce the above copyright
19// notice, this list of conditions and the following disclaimer in the
20// documentation and/or other materials provided with the distribution.
21//
22// 3. Neither the name of the Corporation nor the names of the
23// contributors may be used to endorse or promote products derived from
24// this software without specific prior written permission.
25//
26// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37//
38// Questions? Contact Karen Devine (kddevin@sandia.gov)
39// Erik Boman (egboman@sandia.gov)
40// Siva Rajamanickam (srajama@sandia.gov)
41//
42// ***********************************************************************
43//
44// @HEADER
45
53/**************************************************************/
54/* Includes */
55/**************************************************************/
56
62
63// Teuchos includes
64#include "Teuchos_RCP.hpp"
65#include "Teuchos_XMLParameterListHelpers.hpp"
66#include "Teuchos_Hashtable.hpp"
67
68// SCOREC includes
69#ifdef HAVE_ZOLTAN2_PARMA
70#include <parma.h>
71#include <apf.h>
72#include <apfMesh.h>
73#include <apfMDS.h>
74#include <apfMesh2.h>
75#include <PCU.h>
76#include <gmi_mesh.h>
77#endif
78
79#include <set>
80
81using Teuchos::ParameterList;
82using Teuchos::RCP;
83using Teuchos::ArrayView;
84
85// Computes and prints ghost metrics (takes in a Hyper graph model)
86template <typename Adapter>
88 typedef typename Adapter::gno_t gno_t;
89 typedef typename Adapter::lno_t lno_t;
90 typedef typename Adapter::scalar_t scalar_t;
91 typedef typename Adapter::offset_t offset_t;
93
94 ArrayView<const gno_t> Ids;
95 ArrayView<input_t> wgts;
96 mdl.getEdgeList(Ids,wgts);
97 ArrayView<bool> isOwner;
98 mdl.getOwnedList(isOwner);
99 size_t numOwned = mdl.getLocalNumOwnedVertices();
100 ArrayView<const gno_t> pins;
101 ArrayView<const offset_t> offsets;
102 mdl.getPinList(pins,offsets,wgts);
103
104 std::set<gno_t> gids;
105 for (size_t i=0;i<mdl.getLocalNumVertices();i++) {
106 if (isOwner[i])
107 gids.insert(Ids[i]);
108 }
109 std::set<gno_t> ghosts;
110 gno_t num_ghosts=0;
111 for (size_t i=0;i<mdl.getLocalNumPins();i++) {
112 gno_t pin = pins[i];
113 if (gids.find(pin)==gids.end()) {
114 num_ghosts++;
115 if (ghosts.find(pin)==ghosts.end())
116 ghosts.insert(pin);
117 }
118 }
119 std::cout<< "[METRIC] " << PCU_Comm_Self() << " Total number of ghosts in the hypergraph: " << num_ghosts << "\n"
120 << "[METRIC] " << PCU_Comm_Self() << " Number of unique ghosts: " << ghosts.size() << "\n";
121 gno_t unique_ghosts =ghosts.size();
122 gno_t owned_and_ghosts =unique_ghosts+numOwned;
123 gno_t max_o_and_g,min_o_and_g;
124 gno_t max_ghosts,max_u_ghosts;
125 gno_t min_ghosts,min_u_ghosts;
126 max_ghosts = min_ghosts = num_ghosts;
127 max_u_ghosts = min_u_ghosts = unique_ghosts;
128 max_o_and_g = min_o_and_g = owned_and_ghosts;
129 double avg_ghosts,avg_u_ghosts,avg_o_and_g;
130 PCU_Add_Ints(&num_ghosts,1);
131 PCU_Add_Ints(&unique_ghosts,1);
132 PCU_Add_Ints(&owned_and_ghosts,1);
133 PCU_Max_Ints(&max_ghosts,1);
134 PCU_Max_Ints(&max_u_ghosts,1);
135 PCU_Max_Ints(&max_o_and_g,1);
136 PCU_Min_Ints(&min_ghosts,1);
137 PCU_Min_Ints(&min_u_ghosts,1);
138 PCU_Min_Ints(&min_o_and_g,1);
139 avg_ghosts = num_ghosts*1.0/PCU_Comm_Peers();
140 avg_u_ghosts = unique_ghosts*1.0/PCU_Comm_Peers();
141 avg_o_and_g = owned_and_ghosts*1.0/PCU_Comm_Peers();
142 if (!PCU_Comm_Self())
143 std::cout<< "[METRIC] Global ghosts in the hypergraph (tot max min avg imb): "
144 << num_ghosts<<" "<<max_ghosts<<" "<<min_ghosts<<" "<<avg_ghosts<<" "
145 <<max_ghosts/avg_ghosts << "\n"
146 << "[METRIC] Global unique ghosts (tot max min avg imb): "
147 << unique_ghosts<<" "<<max_u_ghosts<<" "<<min_u_ghosts<<" "<<avg_u_ghosts<<" "
148 <<max_u_ghosts/avg_u_ghosts << "\n"
149 << "[METRIC] Global owned and ghosts (tot max min avg imb): "
150 << owned_and_ghosts<<" "<<max_o_and_g<<" "<<min_o_and_g<<" "<<avg_o_and_g<<" "
151 <<max_o_and_g/avg_o_and_g << "\n";
152
153}
154
155/*****************************************************************************/
156/******************************** MAIN ***************************************/
157/*****************************************************************************/
158
159int main(int narg, char *arg[]) {
160
161 Tpetra::ScopeGuard tscope(&narg, &arg);
162 Teuchos::RCP<const Teuchos::Comm<int> > CommT = Tpetra::getDefaultComm();
163
164 int me = CommT->getRank();
165 //int numProcs = CommT->getSize();
166
167 if (me == 0){
168 std::cout
169 << "====================================================================\n"
170 << "| |\n"
171 << "| Example: Partition APF Mesh |\n"
172 << "| |\n"
173 << "| Questions? Contact Karen Devine (kddevin@sandia.gov), |\n"
174 << "| Erik Boman (egboman@sandia.gov), |\n"
175 << "| Siva Rajamanickam (srajama@sandia.gov). |\n"
176 << "| |\n"
177 << "| Zoltan2's website: http://trilinos.sandia.gov/packages/zoltan2 |\n"
178 << "| Trilinos website: http://trilinos.sandia.gov |\n"
179 << "| |\n"
180 << "====================================================================\n";
181 }
182
183
184#ifdef HAVE_MPI
185 if (me == 0) {
186 std::cout << "PARALLEL executable \n";
187 }
188#else
189 if (me == 0) {
190 std::cout << "SERIAL executable \n";
191 }
192#endif
193
194 /***************************************************************************/
195 /******************************* GET INPUTS ********************************/
196 /***************************************************************************/
197
198 // default values for command-line arguments
199 std::string meshFileName("4/");
200 std::string modelFileName("torus.dmg");
201 std::string action("parma");
202 std::string parma_method("VtxElm");
203 std::string output_loc("");
204 int nParts = CommT->getSize();
205 double imbalance = 1.1;
206 int layers=2;
207 int ghost_metric=false;
208 // Read run-time options.
209 Teuchos::CommandLineProcessor cmdp (false, false);
210 cmdp.setOption("meshfile", &meshFileName,
211 "Mesh file with APF specifications (.smb file(s))");
212 cmdp.setOption("modelfile", &modelFileName,
213 "Model file with APF specifications (.dmg file)");
214 cmdp.setOption("action", &action,
215 "Method to use: mj, scotch, zoltan_rcb, parma or color");
216 cmdp.setOption("parma_method", &parma_method,
217 "Method to use: Vertex, Element, VtxElm, VtxEdgeElm, Ghost, Shape, or Centroid ");
218 cmdp.setOption("nparts", &nParts,
219 "Number of parts to create");
220 cmdp.setOption("imbalance", &imbalance,
221 "Target imbalance for the partitioning method");
222 cmdp.setOption("output", &output_loc,
223 "Location of new partitioned apf mesh. Ex: 4/torus.smb");
224 cmdp.setOption("layers", &layers,
225 "Number of layers for ghosting");
226 cmdp.setOption("ghost_metric", &ghost_metric,
227 "0 does not compute ghost metric otherwise compute both before and after");
228 cmdp.parse(narg, arg);
229
230 /***************************************************************************/
231 /********************** GET CELL TOPOLOGY **********************************/
232 /***************************************************************************/
233
234 // Get dimensions
235 //int dim = 3;
236
237 /***************************************************************************/
238 /***************************** GENERATE MESH *******************************/
239 /***************************************************************************/
240
241#ifdef HAVE_ZOLTAN2_PARMA
242
243 if (me == 0) std::cout << "Generating mesh ... \n\n";
244
245 //Setup for SCOREC
246 PCU_Comm_Init();
247
248 // Generate mesh with MDS
249 gmi_register_mesh();
250 apf::Mesh2* m = apf::loadMdsMesh(modelFileName.c_str(),meshFileName.c_str());
251 apf::verify(m);
252
253 //Data for APF MeshAdapter
254 std::string primary="region";
255 std::string adjacency="face";
256 if (m->getDimension()==2) {
257 primary="face";
258 adjacency="edge";
259 }
260 bool needSecondAdj=false;
261
262 // Set parameters for partitioning
263 if (me == 0) std::cout << "Creating parameter list ... \n\n";
264
265 Teuchos::ParameterList params("test params");
266 params.set("timer_output_stream" , "std::cout");
267
268 bool do_partitioning = false;
269 if (action == "mj") {
270 do_partitioning = true;
271 params.set("debug_level", "basic_status");
272 params.set("imbalance_tolerance", imbalance);
273 params.set("num_global_parts", nParts);
274 params.set("algorithm", "multijagged");
275 params.set("rectilinear", "yes");
276 }
277 else if (action == "scotch") {
278 do_partitioning = true;
279 params.set("debug_level", "no_status");
280 params.set("imbalance_tolerance", imbalance);
281 params.set("num_global_parts", nParts);
282 params.set("partitioning_approach", "partition");
283 params.set("objects_to_partition","mesh_elements");
284 params.set("algorithm", "scotch");
285 needSecondAdj=true;
286 }
287 else if (action == "zoltan_rcb") {
288 do_partitioning = true;
289 params.set("debug_level", "verbose_detailed_status");
290 params.set("imbalance_tolerance", imbalance);
291 params.set("num_global_parts", nParts);
292 params.set("partitioning_approach", "partition");
293 params.set("algorithm", "zoltan");
294 }
295 else if (action == "parma") {
296 do_partitioning = true;
297 params.set("debug_level", "no_status");
298 params.set("imbalance_tolerance", imbalance);
299 params.set("algorithm", "parma");
300 Teuchos::ParameterList &pparams = params.sublist("parma_parameters",false);
301 pparams.set("parma_method",parma_method);
302 pparams.set("step_size",1.1);
303 if (parma_method=="Ghost") {
304 pparams.set("ghost_layers",layers);
305 pparams.set("ghost_bridge",m->getDimension()-1);
306 }
307 adjacency="vertex";
308 }
309 else if (action=="zoltan_hg") {
310 do_partitioning = true;
311 params.set("debug_level", "no_status");
312 params.set("imbalance_tolerance", imbalance);
313 params.set("algorithm", "zoltan");
314 params.set("num_global_parts", nParts);
315 Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
316 zparams.set("LB_METHOD","HYPERGRAPH");
317 zparams.set("LB_APPROACH","PARTITION");
318 //params.set("compute_metrics","yes");
319 adjacency="vertex";
320 }
321 else if (action=="hg_ghost") {
322 do_partitioning = true;
323 params.set("debug_level", "no_status");
324 params.set("imbalance_tolerance", imbalance);
325 params.set("algorithm", "zoltan");
326 params.set("num_global_parts", nParts);
327 params.set("hypergraph_model_type","ghosting");
328 params.set("ghost_layers",layers);
329 Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
330 zparams.set("LB_METHOD","HYPERGRAPH");
331 zparams.set("LB_APPROACH","PARTITION");
332 zparams.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
333 primary="vertex";
334 adjacency="edge";
335 needSecondAdj=true;
336 }
337 else if (action == "color") {
338 params.set("debug_level", "verbose_detailed_status");
339 params.set("debug_output_file", "kdd");
340 params.set("debug_procs", "all");
341 }
342 Parma_PrintPtnStats(m,"before");
343
344 // Creating mesh adapter
345 if (me == 0) std::cout << "Creating mesh adapter ... \n\n";
346 typedef Zoltan2::APFMeshAdapter<apf::Mesh2*> inputAdapter_t;
348 typedef Zoltan2::MeshAdapter<apf::Mesh2*> baseMeshAdapter_t;
349
350 double time_1=PCU_Time();
351 inputAdapter_t *ia =
352 new inputAdapter_t(*CommT, m,primary,adjacency,needSecondAdj);
353 double time_2=PCU_Time();
354
355
356 inputAdapter_t::scalar_t* arr =
357 new inputAdapter_t::scalar_t[ia->getLocalNumOf(ia->getPrimaryEntityType())];
358 for (size_t i=0;i<ia->getLocalNumOf(ia->getPrimaryEntityType());i++) {
359 arr[i]=PCU_Comm_Self()+1;
360 }
361
362 const inputAdapter_t::scalar_t* weights=arr;
363 ia->setWeights(ia->getPrimaryEntityType(),weights,1);
364
365
366 if (ghost_metric) {
367 const baseMeshAdapter_t *base_ia = dynamic_cast<const baseMeshAdapter_t*>(ia);
368 Zoltan2::modelFlag_t graphFlags_;
369 RCP<Zoltan2::Environment> env;
370 try{
371 env = rcp(new Zoltan2::Environment(params, Tpetra::getDefaultComm()));
372 }
374
375 RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<const Zoltan2::Environment>(env);
376
377 RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,false);
378 Zoltan2::HyperGraphModel<inputAdapter_t> model(baseInputAdapter_,envConst,CommT,
379 graphFlags_,Zoltan2::HYPEREDGE_CENTRIC);
380 PrintGhostMetrics(model);
381 }
382
383 // create Partitioning problem
384 double time_3 = PCU_Time();
385 if (do_partitioning) {
386 if (me == 0) std::cout << "Creating partitioning problem ... \n\n";
387
388 Zoltan2::PartitioningProblem<inputAdapter_t> problem(ia, &params, CommT);
389
390 // call the partitioner
391 if (me == 0) std::cout << "Calling the partitioner ... \n\n";
392
393 problem.solve();
394
395
396
397 if (me==0) std::cout << "Applying Solution to Mesh\n\n";
398 apf::Mesh2** new_mesh = &m;
399 ia->applyPartitioningSolution(m,new_mesh,problem.getSolution());
400
401 // create metric object
402 RCP<quality_t> metricObject =
403 rcp(new quality_t(ia, &params, CommT, &problem.getSolution()));
404
405 if (!me) {
406 metricObject->printMetrics(std::cout);
407 }
408 }
409 else {
410 if (me == 0) std::cout << "Creating coloring problem ... \n\n";
411
412 Zoltan2::ColoringProblem<inputAdapter_t> problem(ia, &params);
413
414 // call the partitioner
415 if (me == 0) std::cout << "Calling the coloring algorithm ... \n\n";
416
417 problem.solve();
418
419 problem.printTimers();
420
421
422 }
423
424 double time_4=PCU_Time();
425
426 //Destroy the adapter
427 ia->destroy();
428 delete [] arr;
429 //Parma_PrintPtnStats(m,"after");
430
431 if (ghost_metric) {
432 inputAdapter_t ia2(*CommT, m,primary,adjacency,true);
433 const baseMeshAdapter_t *base_ia = dynamic_cast<const baseMeshAdapter_t*>(&ia2);
434
435 Zoltan2::modelFlag_t graphFlags_;
436 RCP<Zoltan2::Environment> env;
437 try{
438 env = rcp(new Zoltan2::Environment(params, Tpetra::getDefaultComm()));
439 }
441 RCP<const Zoltan2::Environment> envConst = Teuchos::rcp_const_cast<const Zoltan2::Environment>(env);
442 RCP<const baseMeshAdapter_t> baseInputAdapter_(base_ia,false);
443 Zoltan2::HyperGraphModel<inputAdapter_t> model(baseInputAdapter_, envConst, CommT,
444 graphFlags_,Zoltan2::HYPEREDGE_CENTRIC);
445
446 PrintGhostMetrics(model);
447 ia2.destroy();
448 }
449
450 if (output_loc!="") {
451 m->writeNative(output_loc.c_str());
452 }
453
454 // delete mesh
455 if (me == 0) std::cout << "Deleting the mesh ... \n\n";
456 time_4-=time_3;
457 time_2-=time_1;
458 PCU_Max_Doubles(&time_2,1);
459 PCU_Max_Doubles(&time_4,1);
460 if (!me) {
461 std::cout<<"\nConstruction time: "<<time_2<<"\n"
462 <<"Problem time: " << time_4<<"\n\n";
463 }
464 //Delete the APF Mesh
465 m->destroyNative();
466 apf::destroyMesh(m);
467 //End communications
468 PCU_Comm_Free();
469#endif
470
471 if (me == 0)
472 std::cout << "PASS" << std::endl;
473
474 return 0;
475
476}
477/*****************************************************************************/
478/********************************* END MAIN **********************************/
479/*****************************************************************************/
void PrintGhostMetrics(Zoltan2::HyperGraphModel< Adapter > &mdl)
#define nParts
Defines the APFMeshAdapter class.
Defines the ColoringProblem class.
Defines the Environment class.
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
Defines the HyperGraphModel interface.
Defines the PartitioningProblem class.
int main()
ColoringProblem sets up coloring problems for the user.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
The user parameters, debug, timing and memory profiling output objects, and error checking methods.
A class that computes and returns quality metrics.
HyperGraphModel defines the interface required for hyper graph models.
MeshAdapter defines the interface for mesh input.
PartitioningProblem sets up partitioning problems for the user.
const PartitioningSolution< Adapter > & getSolution()
Get the solution to the problem.
void solve(bool updateInputData=true)
Direct the problem to create a solution.
void printTimers() const
Return the communicator passed to the problem.
The StridedData class manages lists of weights or coordinates.
lno_t size() const
Return the length of the strided array.
map_t::local_ordinal_type lno_t
map_t::global_ordinal_type gno_t
std::bitset< NUM_MODEL_FLAGS > modelFlag_t
static ArrayRCP< ArrayRCP< zscalar_t > > weights
Zoltan2::EvaluatePartition< matrixAdapter_t > quality_t