38#ifdef GECODE_HAS_MTRACE
73 tags.
add(description.tag);
79 tag_set(
const char* name) {
81 if (!strcmp(name, description.name))
83 if (!strcmp(name,
"all"))
90 print_tags(std::ostream& os) {
91 for (
const TestTagDescription& description : test_tag_descriptions)
92 os << description.name << std::endl;
100 if (!choices.empty())
102 choices += description.name;
104 return choices +
"|all";
112 if (tags.overlaps(
TestTags(description.tag))) {
115 s += description.name;
129 : _name(
std::move(s)), _tags(t), _next(_tests),
_rand(
Gecode::Support::RandomGenerator()) {
130 _tests =
this; _n_tests++;
133 Base* Base::_tests =
nullptr;
134 unsigned int Base::_n_tests = 0;
149 for (
Base* t = _tests; t !=
nullptr; t = t->next())
155 for ( ; i < _n_tests; i++) {
156 b[i]->next(_tests); _tests = b[i];
167 void report_error(
const std::string& name,
unsigned int seed,
const Options& options, std::ostream& ostream) {
168 ostream <<
"Options: -replay " << seed;
170 ostream <<
" -fixprob " << options.fixprob;
171 ostream <<
" -test " << name << std::endl;
173 ostream <<
olog.str();
180 if (!strcmp(argv[i],
"-help") || !strcmp(argv[i],
"--help")) {
181 std::cerr <<
"Options for testing:" << std::endl
182 <<
"\t-threads (unsigned int) default: " <<
threads << std::endl
183 <<
"\t\tnumber of threads to use. If 0, as many threads as there are cores are used.\n"
184 <<
"\t\tThreaded execution and logging can not be used at the same time."
186 <<
"\t-seed (unsigned int or \"time\") default: "
188 <<
"\t\tseed for random number generator (unsigned int),"
190 <<
"\t\tor \"time\" for a random seed based on "
191 <<
"current time" << std::endl
192 <<
"\t-replay (unsigned int)\n"
193 <<
"\t\treplay the exact -test name once, on one thread\n"
194 <<
"\t-fixprob (unsigned int) default: "
196 <<
"\t\t1/fixprob is the probability of computing a fixpoint"
198 <<
"\t-iter (unsigned int) default: " <<
iter<< std::endl
199 <<
"\t\tthe number of iterations" << std::endl
200 <<
"\t-test (string) default: (none)" << std::endl
201 <<
"\t\tsimple pattern for the tests to run" << std::endl
202 <<
"\t\tprefixing with \"-\" negates the pattern" << std::endl
203 <<
"\t\tprefixing with \"^\" requires a match at the beginning" << std::endl
204 <<
"\t\tmultiple pattern-options may be given"
206 <<
"\t-tag (" << tag_choices() <<
") default: (none)" << std::endl
207 <<
"\t\ttag for the tests to run" << std::endl
208 <<
"\t\tmultiple tag-options may be given"
210 <<
"\t-start (string) default: (none)" << std::endl
211 <<
"\t\tsimple pattern for the first test to run" << std::endl
214 <<
"\t\tlog execution of tests"
216 <<
"\t\tthe optional argument determines the style of the log"
218 <<
"\t\twith text as the default style"
220 <<
"\t-stop (boolean) default: "
221 << (
stop ?
"true" :
"false") << std::endl
222 <<
"\t\tstop on first error or continue" << std::endl
223 <<
"\t-list" << std::endl
224 <<
"\t\toutput list of all test cases and exit" << std::endl
225 <<
"\t-list-tags" << std::endl
226 <<
"\t\toutput list of known test tags and exit" << std::endl
227 <<
"\t-list-with-tags" << std::endl
228 <<
"\t\toutput list of all test cases with tags and exit" << std::endl
231 }
else if (!strcmp(argv[i],
"-threads")) {
232 if (++i == argc)
goto missing;
233 unsigned int argument =
static_cast<unsigned int>(atoi(argv[i]));
239 }
else if (!strcmp(argv[i],
"-seed") || !strcmp(argv[i],
"-replay")) {
240 replay = !strcmp(argv[i],
"-replay");
241 if (++i == argc)
goto missing;
242 if (!strcmp(argv[i],
"time")) {
243 seed =
static_cast<unsigned int>(time(
nullptr));
245 seed =
static_cast<unsigned int>(atoi(argv[i]));
247 }
else if (!strcmp(argv[i],
"-iter")) {
248 if (++i == argc)
goto missing;
249 iter =
static_cast<unsigned int>(atoi(argv[i]));
250 }
else if (!strcmp(argv[i],
"-fixprob")) {
251 if (++i == argc)
goto missing;
252 fixprob =
static_cast<unsigned int>(atoi(argv[i]));
253 }
else if (!strcmp(argv[i],
"-test")) {
254 if (++i == argc)
goto missing;
255 if (argv[i][0] ==
'^')
257 else if (argv[i][0] ==
'-')
261 }
else if (!strcmp(argv[i],
"-tag")) {
262 if (++i == argc)
goto missing;
265 std::cerr <<
"Erroneous argument (-tag)" << std::endl
266 <<
" unknown tag: " << argv[i] << std::endl;
270 }
else if (!strcmp(argv[i],
"-start")) {
271 if (++i == argc)
goto missing;
273 }
else if (!strcmp(argv[i],
"-log")) {
275 }
else if (!strcmp(argv[i],
"-stop")) {
276 if (++i == argc)
goto missing;
277 if(argv[i][0] ==
't') {
279 }
else if (argv[i][0] ==
'f') {
282 }
else if (!strcmp(argv[i],
"-list")) {
284 }
else if (!strcmp(argv[i],
"-list-tags")) {
286 }
else if (!strcmp(argv[i],
"-list-with-tags")) {
295 std::cerr <<
"-replay requires one exact -test name." << std::endl;
303 std::cerr <<
"Logging and multi threading can not be used jointly." << std::endl;
309 std::cerr <<
"Erroneous argument (" << argv[i-1] <<
")" << std::endl
310 <<
" missing parameter" << std::endl;
318 bool positive_patterns =
false;
319 bool match_found =
false;
320 for (
const auto& type_pattern:
testpat) {
321 const auto& type = type_pattern.first;
322 const auto& pattern = type_pattern.second;
324 if (test_name.find(pattern) != std::string::npos) {
330 positive_patterns =
true;
333 if (((type ==
MT_ANY) && (test_name.find(pattern) != std::string::npos)) ||
334 ((type ==
MT_FIRST) && (test_name.find(pattern) == 0)))
340 if (positive_patterns && match_found) {
343 }
else if (positive_patterns && !match_found) {
362 unsigned int seed = options.replay ? options.seed : test_seed;
364 ostream << test->
name() <<
" ";
367 for (
unsigned int i = options.iter; i--;) {
375 ostream <<
"-" << std::endl;
380 ostream << std::endl;
383 ostream <<
"Exception in \"Gecode::" << e.
what()
385 <<
"Stopping..." << std::endl;
393 const size_t ran = passed + failed;
394 std::cout <<
"\nTest summary: " << selected <<
" selected, "
395 << ran <<
" ran, " << passed <<
" passed, "
396 << failed <<
" failed, " << selected - ran <<
" not run."
405 for (
auto test : tests) {
406 unsigned int test_seed = seed_sequence.
next();
407 if (
run_test(test, test_seed, options, std::cout)) {
416 return failed == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
426 const std::vector<Base*>& tests;
432 std::atomic<size_t> next_tests;
434 std::atomic<size_t> passed;
435 std::atomic<size_t> failed;
437 std::atomic<int> running_threads;
439 std::atomic<bool> execution_done_wait_started;
446 static size_t choose_batch_size(
size_t test_count,
int thread_count) {
447 const int batches_per_thread = 5;
448 std::vector<size_t> batch_sizes = {25, 10, 5, 2};
449 for (
auto batch_size : batch_sizes) {
450 if (test_count > batch_size * thread_count * batches_per_thread) {
459 : tests(tests0), options(options0), output_mutex(),
460 next_tests(0), passed(0), failed(0), running_threads(thread_count),
461 execution_done_wait_started(false), execution_done_event() {}
465 assert(running_threads.load() == 0);
467 return failed.load() == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
478 execution_done_wait_started.exchange(
true);
479 assert(some_waiting ==
false &&
"Only one thread is allowed to await the result");
480 execution_done_event.wait();
484 void record_result(
bool success) {
493 if (running_threads.fetch_sub(1) == 1) {
494 execution_done_event.
signal();
500 std::pair<size_t, size_t> next_batch() {
501 const size_t current_start = next_tests.load();
502 const size_t batch_size = choose_batch_size(tests.size() - current_start, running_threads);
503 const size_t next_start = next_tests.fetch_add(batch_size);
504 return std::make_pair(next_start, batch_size);
508 bool continue_testing() {
510 return !options.stop || failed.load() == 0;
514 void write_output(std::string output) {
515 Gecode::Support::Lock lock(output_mutex);
528 const int initial_seed;
532 : tec(tec), initial_seed(initialSeed) {}
541 const std::pair<size_t, size_t>& start_and_size = tec.next_batch();
542 const size_t batch_start = start_and_size.first;
543 const size_t batch_size = start_and_size.second;
544 if (batch_start >= tec.tests.size()) {
549 for (
size_t i = batch_start; i < batch_start + batch_size && i < tec.tests.
size(); ++i) {
550 if (!tec.continue_testing()) {
553 auto test = tec.tests[i];
554 unsigned int test_seed = seed_sequence.
next();
555 std::ostringstream test_output;
556 tec.record_result(
run_test(test, test_seed, tec.options, test_output));
557 tec.write_output(test_output.str());
559 if (!tec.continue_testing()) {
576 for (
unsigned int i = 0; i < options.threads; ++i) {
585 std::vector<Base*> tests;
586 bool started = options.start_from ==
nullptr;
589 if (t->name().find(options.start_from) != std::string::npos) {
595 if (options.is_test_name_matching(t->name()) &&
596 options.is_test_tags_matching(t->tags())) {
597 tests.emplace_back(t);
605 opt.parse(argc, argv);
608 print_tags(std::cout);
614 if (
opt.list ||
opt.list_with_tags) {
616 std::cout << t->name();
617 if (
opt.list_with_tags)
618 std::cout <<
" [" << tags_to_string(t->tags()) <<
"]";
619 std::cout << std::endl;
625 if (
opt.replay &&
tests.size() != 1) {
626 std::cerr <<
"-replay requires exactly one registered test." << std::endl;
631 const int result =
opt.threads > 1 ?
633 std::ostringstream elapsed;
634 elapsed << std::fixed << std::setprecision(3) << timer.
stop() / 1000.0;
635 std::cout <<
"Elapsed time: " << elapsed.str() <<
" s." << std::endl;
642 for (
int j=i.l; j--; )
int size(void) const
Return size of array (number of elements).
Exception: Base-class for exceptions
virtual const char * what(void) const noexcept
Return information.
An event for synchronization.
void signal(void)
Signal the event.
unsigned int next(void)
Generate next number in series.
void seed(unsigned int s)
Set the current seed to s.
A mutex for mutual exclausion among several threads.
An interface for objects that can be run by a thread.
static void run(Runnable *r)
Construct a new thread and run r.
static unsigned int npu(void)
Return number of processing units (1 if information not available).
double stop(void)
Get time since start of timer.
void start(void)
Start timer.
Base class for all tests to be run
Gecode::Support::RandomGenerator _rand
Random number generator.
Base(std::string s)
Create and register a standard test with name s.
static Base * tests(void)
Return all tests.
virtual ~Base(void)
Destructor.
virtual bool run(void)=0
Run test.
const std::string & name(void) const
Return name of test.
static void sort(void)
Sort tests alphabetically.
bool list_tags
Whether to list known tags.
std::vector< std::pair< MatchType, const char * > > testpat
Patterns to test against.
bool is_test_name_matching(const std::string &test_name) const
Test whether a test name should be executed.
void parse(int argc, char *argv[])
Parse commandline arguments.
bool log
Whether to log the tests.
unsigned int seed
The random seed to be used.
const char * start_from
Name of first test to start with.
bool is_test_tags_matching(TestTags tags) const
True iff test tags should be executed according to the requested tags. With no tag request,...
static const unsigned int deffixprob
Default fixpoint probability.
bool replay
Replay one named test from its reported random state.
bool list
Whether to list all tests.
unsigned int iter
Number of iterations for each test.
bool stop
Whether to stop on an error.
unsigned int threads
Number of threads to use.
unsigned int fixprob
The probability for computing a fixpoint.
static const int defiter
Default number of iterations.
TestTags testtags
Tags to test against.
bool list_with_tags
Whether to include tags when listing tests.
bool operator()(Base *x, Base *y)
friend class TestExecutor
int report_result()
Report the results after all threads have finished and return the exit status.
TestExecutionControl(const std::vector< Base * > &tests0, const Options &options0, int thread_count)
void await_test_runners_completed()
TestExecutor(TestExecutionControl &tec, const int initialSeed)
void run(void) override
The function that is executed when the thread starts.
Simple class for describing indentation.
Heap heap
The single global heap.
LinearCongruentialGenerator< 2147483647, 48271, 44488, 3399 > RandomGenerator
Default values for linear congruential generator.
Support algorithms and datastructures
void quicksort(Type *l, Type *r, Less &less)
Standard quick sort.
Gecode toplevel namespace
Case output("Data::Output", [](void) { Data d=read("n: int f: float w: word m[2][2]: int repeat n { k: int x[k]: int }", "2 20 a 1 2 3 4 1 5 2 6 7");std::ostringstream z;d.dzn(z);CHECK(z.str().find("f = 20.0;") !=std::string::npos);CHECK(z.str().find("m = [| 1, 2\n | 3, 4 |];") !=std::string::npos);CHECK(z.str().find("% x: array[int] of tuple(array[int] of int)\n" "x = [([5],), ([6, 7],)];") !=std::string::npos);CHECK(z.str().find("w = \"a\";") !=std::string::npos);std::ostringstream zq;read("var: int w: word", "3 a\033b").dzn(zq);CHECK(zq.str().find("'var' = 3;") !=std::string::npos);CHECK(zq.str().find("w = \"a\\x1bb\";") !=std::string::npos);Data e=read("e[2][0]: int repeat 2 { k: int repeat k { list y: int until 0 } }", "1 7 0 2 8 9 0 0");std::ostringstream ze;e.dzn(ze);CHECK(ze.str().find("e = array2d(1..2, 1..0, []);") !=std::string::npos);CHECK(ze.str().find("y = [([([7],)],), ([([8, 9],), ([],)],)];") !=std::string::npos);std::ostringstream z7;read("a[1][1][1][1][1][1][1]: int", "5").dzn(z7);CHECK(z7.str().find("a = [([([([([([([5],)],)],)],)],)],)];") !=std::string::npos);return true;})
void print_test_summary(size_t selected, size_t passed, size_t failed)
Report test cases; iterations of a test count as one case.
std::ostringstream olog
Stream used for logging.
std::vector< Base * > select_tests(const Options &options)
int run_tests(const std::vector< Base * > &tests, const Options &options)
Run all the tests with the supplied options.
int run_registered_tests(int argc, char *argv[])
bool run_test(Base *test, unsigned int test_seed, const Options &options, std::ostream &ostream)
Run a single test, returning true iff the test succeeded.
TestTag
Tags for test selection.
@ check
Basic integrity tests.
@ standard
Standard test suite.
@ sweep
Really heavy sweep tests.
void report_error(const std::string &name, unsigned int seed, const Options &options, std::ostream &ostream)
int run_tests_parallel(const std::vector< Base * > &tests, const Options &options)
Run all the tests with the supplied options in parallel.
A command-line name and its corresponding test tag.
std::ostream & operator<<(std::ostream &os, const Test::ind &i)