1 /** 2 Unit test runner with structured result output support. 3 4 Copyright: Copyright ©2013 rejectedsoftware e.K., all rights reserved. 5 Authors: Sönke Ludwig 6 */ 7 module tested; 8 9 import core.sync.condition; 10 import core.sync.mutex; 11 import core.memory; 12 import core.time; 13 import core.thread; 14 import std.datetime : StopWatch; 15 import std..string : startsWith; 16 import std.traits; 17 import std.typetuple : TypeTuple; 18 19 20 /** 21 Runs all unit tests contained in the given symbol recursively. 22 23 COMPOSITES can be a list of modules or composite types. 24 25 Example: 26 This example assumes that the application has all of its sources in 27 the package named "mypackage". All contained tests will be run 28 recursively. 29 30 --- 31 import tested; 32 import mypackage.application; 33 34 void main() 35 { 36 version(unittest) runUnitTests!(mypackage.application)(new ConsoleResultWriter); 37 else runApplication; 38 } 39 --- 40 */ 41 bool runUnitTests(COMPOSITES...)(TestResultWriter results) 42 { 43 assert(!g_runner, "Running multiple unit tests concurrently is not supported."); 44 g_runner = new TestRunner(results); 45 auto ret = g_runner.runUnitTests!COMPOSITES(); 46 g_runner = null; 47 return ret; 48 } 49 50 51 /** 52 Emits a custom instrumentation value that will be stored in the unit test results. 53 */ 54 void instrument(string name, double value) 55 { 56 if (g_runner) g_runner.instrument(name, value); 57 } 58 59 60 /** 61 Attribute for giving a unit test a name. 62 63 The name of a unit test will be output in addition to the fully 64 qualified name of the test function in the test output. 65 66 Example: 67 --- 68 @name("some test") 69 unittest { 70 // unit test code 71 } 72 73 @name("some other test") 74 unittest { 75 // unit test code 76 } 77 --- 78 */ 79 struct name { string name; } 80 81 82 /** 83 Base interface for all unit test result writers. 84 */ 85 interface TestResultWriter { 86 void finalize(); 87 void beginTest(string name, string qualified_name); 88 void addScalar(Duration timestamp, string name, double value); 89 void endTest(Duration timestamp, Throwable error); 90 } 91 92 93 /** 94 Directly outputs unit test results to the console. 95 */ 96 class ConsoleTestResultWriter : TestResultWriter { 97 import std.stdio; 98 private { 99 size_t m_failCount, m_successCount; 100 string m_name, m_qualifiedName; 101 } 102 103 void finalize() 104 { 105 writefln("==========================="); 106 writefln("%s of %s tests have passed.", m_successCount, m_successCount+m_failCount); 107 writefln(""); 108 writefln("FINAL RESULT: %s", m_failCount > 0 ? "FAILED" : "PASSED"); 109 } 110 111 void beginTest(string name, string qualified_name) 112 { 113 m_name = name; 114 m_qualifiedName = qualified_name; 115 } 116 117 void addScalar(Duration timestamp, string name, double value) 118 { 119 } 120 121 void endTest(Duration timestamp, Throwable error) 122 { 123 if (error) { 124 writefln(`FAIL "%s" (%s) after %.6f s: %s`, m_name, m_qualifiedName, fracSecs(timestamp), error.msg); 125 m_failCount++; 126 } else { 127 writefln(`PASS "%s" (%s) after %.6f s`, m_name, m_qualifiedName, fracSecs(timestamp)); 128 m_successCount++; 129 } 130 } 131 132 static double fracSecs(Duration dur) 133 { 134 return 1E-6 * dur.total!"usecs"; 135 } 136 } 137 138 /** 139 Outputs test results and instrumentation values 140 */ 141 class JsonTestResultWriter : TestResultWriter { 142 import std.stdio; 143 144 private { 145 File m_file; 146 bool m_gotInstrumentValue, m_gotUnitTest; 147 } 148 149 this(string filename) 150 { 151 m_file = File(filename, "w+b"); 152 m_file.writeln("["); 153 m_gotUnitTest = false; 154 } 155 156 void finalize() 157 { 158 m_file.writeln(); 159 m_file.writeln("]"); 160 m_file.close(); 161 } 162 163 void beginTest(string name, string qualified_name) 164 { 165 if (m_gotUnitTest) m_file.writeln(","); 166 else m_gotUnitTest = true; 167 m_file.writef(`{"name": "%s", "qualifiedName": "%s", "instrumentation": [`, name, qualified_name); 168 m_gotInstrumentValue = false; 169 } 170 171 void addScalar(Duration timestamp, string name, double value) 172 { 173 if (m_gotInstrumentValue) m_file.write(", "); 174 else m_gotInstrumentValue = true; 175 m_file.writef(`{"name": "%s", "value": %s, "timestamp": %s}`, name, value, timestamp.total!"usecs" * 1E-6); 176 } 177 178 void endTest(Duration timestamp, Throwable error) 179 { 180 m_file.writef(`], "success": %s, "duration": %s`, error is null, timestamp.total!"usecs" * 1E-6); 181 if (error) m_file.writef(`, "message": "%s"`, error.msg); 182 m_file.write("}"); 183 } 184 } 185 186 private class TestRunner { 187 private { 188 Mutex m_mutex; 189 Condition m_condition; 190 TestResultWriter m_results; 191 InstrumentStats m_baseStats; 192 bool m_running, m_quit, m_instrumentsReady; 193 StopWatch m_stopWatch; 194 } 195 196 this(TestResultWriter writer) 197 { 198 m_results = writer; 199 m_mutex = new Mutex; 200 m_condition = new Condition(m_mutex); 201 } 202 203 bool runUnitTests(COMPOSITES...)() 204 { 205 InstrumentStats basestats; 206 m_running = false; 207 m_quit = false; 208 m_instrumentsReady = false; 209 210 auto instrumentthr = new Thread(&instrumentThread); 211 instrumentthr.name = "instrumentation thread"; 212 //instrumentthr.priority = Thread.PRIORITY_DEFAULT + 1; 213 instrumentthr.start(); 214 215 auto ret = true; 216 foreach(comp; COMPOSITES) 217 if (!runUnitTestsImpl!comp()) 218 ret = false; 219 m_results.finalize(); 220 221 synchronized (m_mutex) m_quit = true; 222 m_condition.notifyAll(); 223 instrumentthr.join(); 224 225 return ret; 226 } 227 228 private void instrument(string name, double value) 229 { 230 auto ts = cast(Duration)m_stopWatch.peek(); 231 synchronized (m_mutex) m_results.addScalar(ts, name, value); 232 } 233 234 private bool runUnitTestsImpl(COMPOSITE...)() 235 if (COMPOSITE.length == 1 && isUnitTestContainer!COMPOSITE) 236 { 237 bool ret = true; 238 //pragma(msg, fullyQualifiedName!COMPOSITE); 239 240 foreach (test; __traits(getUnitTests, COMPOSITE)) { 241 if (!runUnitTest!test()) 242 ret = false; 243 } 244 245 // if COMPOSITE has members, descent recursively 246 static if (isUnitTestContainer!COMPOSITE) { 247 foreach (M; __traits(allMembers, COMPOSITE)) { 248 static if ( 249 __traits(compiles, __traits(getMember, COMPOSITE, M)) && 250 isSingleField!(__traits(getMember, COMPOSITE, M)) && 251 isUnitTestContainer!(__traits(getMember, COMPOSITE, M)) && 252 !isModule!(__traits(getMember, COMPOSITE, M)) 253 ) 254 { 255 if (!runUnitTestsImpl!(__traits(getMember, COMPOSITE, M))()) 256 ret = false; 257 } 258 } 259 } 260 261 return ret; 262 } 263 264 private bool runUnitTest(alias test)() 265 { 266 string name; 267 foreach (att; __traits(getAttributes, test)) 268 static if (is(typeof(att) == .name)) 269 name = att.name; 270 271 // wait for instrumentation thread to get ready 272 synchronized (m_mutex) 273 while (!m_instrumentsReady) 274 m_condition.wait(); 275 276 m_results.beginTest(name, fullyQualifiedName!test); 277 m_stopWatch.reset(); 278 279 GC.collect(); 280 m_baseStats = InstrumentStats.instrument(); 281 282 synchronized (m_mutex) m_running = true; 283 m_condition.notifyAll(); 284 285 Throwable error; 286 try { 287 m_stopWatch.start(); 288 test(); 289 m_stopWatch.stop(); 290 } catch (Throwable th) { 291 m_stopWatch.stop(); 292 error = th; 293 } 294 295 auto duration = cast(Duration)m_stopWatch.peek(); 296 297 auto stats = InstrumentStats.instrument(); 298 synchronized (m_mutex) { 299 writeInstrumentStats(m_results, duration, m_baseStats, stats); 300 m_running = false; 301 } 302 303 m_results.endTest(duration, error); 304 return error is null; 305 } 306 307 private void instrumentThread() 308 { 309 while (true) { 310 synchronized (m_mutex) { 311 if (m_quit) return; 312 if (!m_running) { 313 m_instrumentsReady = true; 314 m_condition.notifyAll(); 315 while (!m_running && !m_quit) m_condition.wait(); 316 if (m_quit) return; 317 m_instrumentsReady = false; 318 } 319 } 320 321 Thread.sleep(10.msecs); 322 323 auto ts = cast(Duration)m_stopWatch.peek; 324 auto stats = InstrumentStats.instrument(); 325 synchronized (m_mutex) 326 if (m_running) 327 writeInstrumentStats(m_results, ts, m_baseStats, stats); 328 } 329 } 330 } 331 332 333 // copied from gc.stats 334 private struct GCStats { 335 size_t poolsize; // total size of pool 336 size_t usedsize; // bytes allocated 337 size_t freeblocks; // number of blocks marked FREE 338 size_t freelistsize; // total of memory on free lists 339 size_t pageblocks; // number of blocks marked PAGE 340 } 341 342 // from gc.proxy 343 private extern (C) GCStats gc_stats(); 344 345 private struct InstrumentStats { 346 size_t gcPoolSize; 347 size_t gcUsedSize; 348 size_t gcFreeListSize; 349 350 static InstrumentStats instrument() 351 { 352 auto stats = gc_stats(); 353 InstrumentStats ret; 354 ret.gcPoolSize = stats.poolsize; 355 ret.gcUsedSize = stats.usedsize; 356 ret.gcFreeListSize = stats.freelistsize; 357 return ret; 358 } 359 } 360 361 private { 362 __gshared TestRunner g_runner; 363 } 364 365 private void writeInstrumentStats(TestResultWriter results, Duration timestamp, InstrumentStats base_stats, InstrumentStats stats) 366 { 367 results.addScalar(timestamp, "gcPoolSize", stats.gcPoolSize - base_stats.gcPoolSize); 368 results.addScalar(timestamp, "gcUsedSize", stats.gcUsedSize - base_stats.gcUsedSize); 369 results.addScalar(timestamp, "gcFreeListSize", stats.gcFreeListSize - base_stats.gcFreeListSize); 370 } 371 372 private template isUnitTestContainer(DECL...) 373 if (DECL.length == 1) 374 { 375 static if (!isAccessible!DECL) { 376 enum isUnitTestContainer = false; 377 } else static if (is(FunctionTypeOf!(DECL[0]))) { 378 enum isUnitTestContainer = false; 379 } else static if (is(DECL[0]) && !isAggregateType!(DECL[0])) { 380 enum isUnitTestContainer = false; 381 } else static if (isPackage!(DECL[0])) { 382 enum isUnitTestContainer = false; 383 } else static if (isModule!(DECL[0])) { 384 enum isUnitTestContainer = DECL[0].stringof != "module object"; 385 } else static if (!__traits(compiles, fullyQualifiedName!(DECL[0]))) { 386 enum isUnitTestContainer = false; 387 } else static if (!is(typeof(__traits(allMembers, DECL[0])))) { 388 enum isUnitTestContainer = false; 389 } else { 390 enum isUnitTestContainer = true; 391 } 392 } 393 394 private template isModule(DECL...) 395 if (DECL.length == 1) 396 { 397 static if (is(DECL[0])) enum isModule = false; 398 else static if (is(typeof(DECL[0])) && !is(typeof(DECL[0]) == void)) enum isModule = false; 399 else static if (!is(typeof(DECL[0].stringof))) enum isModule = false; 400 else static if (is(FunctionTypeOf!(DECL[0]))) enum isModule = false; 401 else enum isModule = DECL[0].stringof.startsWith("module "); 402 } 403 404 private template isPackage(DECL...) 405 if (DECL.length == 1) 406 { 407 static if (is(DECL[0])) enum isPackage = false; 408 else static if (is(typeof(DECL[0])) && !is(typeof(DECL[0]) == void)) enum isPackage = false; 409 else static if (!is(typeof(DECL[0].stringof))) enum isPackage = false; 410 else static if (is(FunctionTypeOf!(DECL[0]))) enum isPackage = false; 411 else enum isPackage = DECL[0].stringof.startsWith("package "); 412 } 413 414 private template isAccessible(DECL...) 415 if (DECL.length == 1) 416 { 417 enum isAccessible = __traits(compiles, testTempl!(DECL[0])()); 418 } 419 420 private template isSingleField(DECL...) 421 { 422 enum isSingleField = DECL.length == 1; 423 } 424 425 static assert(!is(tested)); 426 static assert(isModule!tested); 427 static assert(!isPackage!tested); 428 static assert(isPackage!std); 429 static assert(__traits(compiles, testTempl!GCStats())); 430 static assert(__traits(compiles, testTempl!(immutable(ubyte)[]))); 431 static assert(isAccessible!GCStats); 432 static assert(isUnitTestContainer!GCStats); 433 static assert(isUnitTestContainer!tested); 434 435 private void testTempl(X...)() 436 if (X.length == 1) 437 { 438 static if (is(X[0])) { 439 auto x = X[0].init; 440 } else { 441 auto x = X[0].stringof; 442 } 443 }