Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Loading...
Searching...
No Matches
Teuchos_UnitTestRepository.cpp
Go to the documentation of this file.
1// @HEADER
2// ***********************************************************************
3//
4// Teuchos: Common Tools Package
5// Copyright (2004) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38//
39// ***********************************************************************
40// @HEADER
41
42
47#include "Teuchos_Array.hpp"
48#include "Teuchos_Assert.hpp"
51#include "Teuchos_Assert.hpp"
52#include "Teuchos_Time.hpp"
54
55
56namespace Teuchos {
57
58
60
62 std::string groupName;
63 std::string testName;
65
68 const std::string groupName_in,
69 const std::string testName_in
70 )
73 {
74#ifdef TEUCHOS_DEBUG
76#endif
77 }
78
79private:
80 UnitTestData(); // Not defined!
82};
83
84
86
87
89{
90 if (a.groupName < b.groupName) {
91 return true;
92 }
93 else if (a.groupName > b.groupName) {
94 return false;
95 }
96 return a.insertionIndex < b.insertionIndex;
97}
98
99
100
101std::string getUnitTestName(const std::string groupName,
102 const std::string testName)
103{
104 std::ostringstream oss;
105 oss << groupName<<"_"<<testName<<"_UnitTest";
106 return oss.str();
107}
108
109
115
116
117bool strMatch( const std::string &fullMatchStr, const std::string &str )
118{
119
120 const std::string::size_type npos = std::string::npos;
121
122 const size_t strLen = str.length();
123 const size_t fullMatchStrLen = fullMatchStr.length();
124
125 if (fullMatchStrLen == 0) {
126 return true;
127 }
128
129 const bool beginGlob = fullMatchStr[0] == '*';
130 const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
131
132 const size_t matchStrLen =
133 fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
134
135 if (matchStrLen == 0) {
136 return true;
137 }
138
139 if (matchStrLen > strLen) {
140 return false;
141 }
142
143 if (beginGlob && endGlob) {
144 return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
145 }
146
147 if (endGlob) {
148 return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
149 }
150
151 if (beginGlob) {
152 return fullMatchStr.substr(1, matchStrLen) ==
154 }
155
156 return fullMatchStr == str;
157
158}
159
160
161} // namespace Teuchos
162
163
164
165
166namespace Teuchos {
167
168
169// Implementation class
170
171
204
205
206// public
207
208
213
214
216 const bool globallyReduceUnitTestResult)
217{
218 getData().globallyReduceUnitTestResult = globallyReduceUnitTestResult;
219}
220
221
223{
224 return getData().globallyReduceUnitTestResult;
225}
226
227
229{
230
231 typedef InstanceData::unitTests_t unitTests_t;
232
233 using std::setprecision;
234
235 Time overallTimer("overallTimer", true);
236 Time timer("timer");
237
238 const int timerPrec = 3;
239
240 out << "\n***\n*** Unit test suite ...\n***\n\n";
241
242 InstanceData &data = getData();
243
244 const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
246
248
249 bool success = true;
250 int testCounter = 0;
251 int numTestsRun = 0;
252 int numTestsFailed = 0;
253
255
256 try {
257
258 out << "\nSorting tests by group name then by the order they were added ...";
259 timer.start(true);
260 std::sort( data.unitTests.begin(), data.unitTests.end() );
261 timer.stop();
262 out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
263
264 out << "\nRunning unit tests ...\n\n";
265 unitTests_t::iterator iter = data.unitTests.begin();
266 for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
267
268 const UnitTestData &utd = (*iter);
269
270 const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
271
272 if (
273 (
274 strMatch(data.groupName, utd.groupName)
275 &&
276 strMatch(data.testName, utd.testName)
277 )
278 &&
279 (
280 data.notUnitTestName.length() == 0
281 ||
283 )
284 )
285 {
286
287 ++numTestsRun;
288
289 std::ostringstream testHeaderOSS;
290 testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
291 const std::string testHeader = testHeaderOSS.str();
292
293 if (showAll)
294 out <<"\n";
295
296 if (showTestNames)
297 out <<testHeader<<std::flush;
298
299 {
300
303 if (showAll) {
304 out << "\n";
305 localOut = rcpFromRef(out);
306 }
307 else {
308 oss = rcp(new std::ostringstream);
310 }
311
312 OSTab tab(out);
313
314 if (!data.noOp) {
315
316 timer.start(true);
317 const bool result = runUnitTestImpl(*utd.unitTest, *localOut);
318 timer.stop();
319
320 if (!result) {
321
322 failedTests.push_back(testHeader);
323
324 if (!showTestNames)
325 out <<testHeader<<"\n"<<std::flush;
326 else if (!showAll)
327 out <<"\n";
328
329 if (!is_null(oss))
330 out << oss->str();
331
332 out
333 <<"[FAILED] "
334 <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
335 <<" "<<unitTestName<<"\n"
336 <<"Location: "<<utd.unitTest->unitTestFile()<<":"
337 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
338
339 if (!is_null(oss))
340 out << "\n";
341
342 success = false;
343
345
346 }
347 else {
348
349 if (showTestNames)
350 out << "[Passed] "
351 << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
352
353 if (showAll && data.showSrcLocation)
354 out
355 << "Location: "<<utd.unitTest->unitTestFile()<<":"
356 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
357
358 }
359
360 }
361 else {
362
363 if (showTestNames)
364 out << "[Not Run]\n";
365
366 }
367
368 }
369
370 }
371
372 }
373
374 TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
375
376 }
378
379 if (failedTests.size()) {
380 out << "\nThe following tests FAILED:\n";
381 for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
382 out << " " << failedTests[i] << "\n";
383 }
384
385 overallTimer.stop();
386 out << "\nTotal Time: " << setprecision(timerPrec)
387 << overallTimer.totalElapsedTime() << " sec\n";
388
389 out
390 << "\nSummary: total = " << testCounter
391 << ", run = " << numTestsRun;
392
393 if (!data.noOp) {
394 out
395 << ", passed = " << (numTestsRun-numTestsFailed)
396 << ", failed = " << numTestsFailed << "\n";
397 }
398 else {
399 out
400 << ", passed = ???"
401 << ", failed = ???\n";
402 }
403
404 return success;
405
406}
407
408
410{
411
413
414 CommandLineProcessor &clp = getData().clp;
415 setUpCLP(outArg(clp));
417 clp.parse(argc,argv);
419 *out << "\nEnd Result: TEST FAILED" << std::endl;
420 return parse_return;
421 }
422
423 const bool success = runUnitTests(*out);
424
425 if (success)
426 *out << "\nEnd Result: TEST PASSED" << std::endl;
427 else
428 *out << "\nEnd Result: TEST FAILED" << std::endl;
429
430 clp.printFinalTimerSummary(out.ptr());
431
432 return (success ? 0 : 1);
433
434}
435
436
438 const std::string groupName, const std::string testName_in )
439{
440 InstanceData &data = getData();
441 std::string testName = testName_in;
442 data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
443}
444
445
447{
448 return (getData().showTestDetails == SHOW_TEST_DETAILS_ALL);
449}
450
451
452// private:
453
454
457
458
460{
461
462 clp->addOutputSetupOptions(true);
463
464 const int numShowTestDetails = 3;
469 };
471 { "ALL",
472 "TEST_NAMES",
473 "FINAL_RESULTS"
474 };
475 clp->setOption(
476 "show-test-details", &getData().showTestDetails,
478 "Level of detail to show in the tests"
479 );
480 clp->setOption(
481 "details", &getData().showTestDetails,
483 "Short for --show-test-details"
484 );
485
486 clp->setOption(
487 "show-src-location", "no-show-src-location", &getData().showSrcLocation,
488 "If true, then the location of the unit test source code is shown."
489 " Only meaningful if --show-test-details=ALL."
490 );
491
492 clp->setOption(
493 "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
494 "If true, then the location of every failed unit test check is printed."
495 );
496
497 clp->setOption(
498 "globally-reduce-test-result", "no-globally-reduce-test-result",
499 &getData().globallyReduceUnitTestResult,
500 "If true, individual unit test pass/fail is globally reduced across MPI processes."
501 );
502
503 clp->setOption(
504 "group-name", &getData().groupName,
505 "If specified, selects only tests that match the group name glob." );
506 clp->setOption(
507 "group", &getData().groupName,
508 "Short for --group-name." );
509
510 clp->setOption(
511 "test-name", &getData().testName,
512 "If specified, selects only tests that match the test name glob." );
513 clp->setOption(
514 "test", &getData().testName,
515 "Short for --test-name." );
516
517 clp->setOption(
518 "not-unit-test", &getData().notUnitTestName,
519 "If specified, full unit tests with glob matches will *not* be run." );
520
521 clp->setOption(
522 "no-op", "do-op", &getData().noOp,
523 "If --no-op, then only the names of the tests that would be run are run."
524 );
525
526}
527
528
534
535
537 FancyOStream &out)
538{
539 const bool result = unitTest.runUnitTest(out);
540 if (getData().globallyReduceUnitTestResult) {
541 const int globalSum = GlobalMPISession::sum(result ? 0 : 1);
542 if (globalSum == 0) {
543 return true;
544 }
545 else {
546 // Only print that there are failures on processes where the local
547 // unit test actally passed. On processes where the local unit test
548 // fails, users already know that test failed so there is no need to
549 // exlain it.
550 if (result) {
551 out << "NOTE: Global reduction shows failures on other processes!\n"
552 << "(rerun with --output-to-root-rank-only=-1 to see output\n"
553 << "from other processes to see what process failed!)\n";
554 }
555 else {
556 // The test failed on the root process so the user already knows it failed!
557 }
558 // Determine what processes have failing tests
563 for ( int proc_k = 0; proc_k < numProcs; ++proc_k ) {
564 if (passFailFlags[proc_k] != 0) {
565 procsThatFailed.push_back(proc_k);
566 }
567 }
568 // Print what processes have the failing tests. If there is only one
569 // processes, don't print anything.
570 if (numProcs > 1) {
571 if (procsThatFailed.size() == numProcs) {
572 out << "NOTE: Unit test failed on all processes!\n";
573 // NOTE: when all the processes are failing it is useless to print
574 // out a list of all of the processes.
575 }
576 else {
577 out << "NOTE: Unit test failed on processes = " << procsThatFailed << "\n"
578 << "(rerun with --output-to-root-rank-only=<procID> to see output\n"
579 << "from individual processes where the unit test is failing!)\n";
580 }
581 }
582 return false;
583 }
584 }
585 return result;
586}
587
588
589} // namespace Teuchos
Templated array class derived from the STL std::vector.
Basic command line parser for input from (argc,argv[])
Teuchos header file which uses auto-configuration information to include necessary C++ headers.
#define TEUCHOS_STANDARD_CATCH_STATEMENTS(VERBOSE, ERR_STREAM, SUCCESS_FLAG)
Simple macro that catches and reports standard exceptions and other exceptions.
Utilities to make writing tests easier.
Basic wall-clock timer class.
Unit testing support.
Unit testing support.
size_type size() const
void push_back(const value_type &x)
int size(const Comm< Ordinal > &comm)
Get the number of processes in the communicator.
Class that helps parse command line input arguments from (argc,argv[]) and set options.
EParseCommandLineReturn
Return value for CommandLineProcessor::parse(). Note: These enums are all given non-negative values s...
EParseCommandLineReturn parse(int argc, char *argv[], std::ostream *errout=&std::cerr) const
Parse a command line.
void printFinalTimerSummary(const Ptr< std::ostream > &out=null)
Call to print timers so that they don't get printed in the destructor.
static int sum(int localVal)
Sum a set of integers across processes.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
Concrete serial communicator subclass.
Wall-clock timer.
Unit test base class.
bool runUnitTest(FancyOStream &out) const
static void setUpCLP(const Ptr< CommandLineProcessor > &clp)
static bool verboseUnitTests()
Returns if unit tests are verbose or not.
static bool runUnitTests(FancyOStream &out)
Run the registered unit tests.
static bool getGloballyReduceTestResult()
Get if the unit tests should reduce across processes or not.
static CommandLineProcessor & getCLP()
Return the CLP to add options to.
static int runUnitTestsFromMain(int argc, char *argv[])
Run the unit tests from main() passing in (argc, argv).
static void addUnitTest(UnitTestBase *unitTest, const std::string groupName, const std::string testName)
Add an unit test (called indirectly through macros.
static bool runUnitTestImpl(const UnitTestBase &unitTest, FancyOStream &out)
static void setGloballyReduceTestResult(const bool globallyReduceUnitTestResult)
Set if the unit tests should reduce pass/fail across processes.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
std::ostream subclass that performs the magic of indenting data sent to an std::ostream object among ...
#define TEUCHOS_ASSERT(assertion_test)
This macro is throws when an assert fails.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
bool operator<(BigUInt< n > const &a, BigUInt< n > const &b)
bool strMatch(const std::string &fullMatchStr, const std::string &str)
std::string getUnitTestName(const std::string groupName, const std::string testName)
bool showTestFailureLocation()
Return if TEUCHOS_PASS_FAIL(...) should print test failure location.
const Teuchos::UnitTestBase * unitTest
UnitTestData(Teuchos::UnitTestBase *unitTest_in, const std::string groupName_in, const std::string testName_in)