1 /** 2 Unit test runner with structured result output support. 3 4 Copyright: Copyright ©2013 rejectedsoftware e.K., all rights reserved. 5 Authors: Sönke Ludwig 6 */ 7 module tested; 8 9 import core.sync.condition; 10 import core.sync.mutex; 11 import core.memory; 12 import core.time; 13 import core.thread; 14 import std.datetime : StopWatch; 15 import std.string : startsWith; 16 import std.traits : isAggregateType, fullyQualifiedName; 17 import std.typetuple : TypeTuple; 18 19 20 /** 21 Runs all unit tests contained in the given symbol recursively. 22 23 composite can be a package, a module or a composite type. 24 25 Example: 26 This example assumes that the application has all of its sources in 27 the package named "mypackage". All contained tests will be run 28 recursively. 29 30 --- 31 import tested; 32 import mypackage.application; 33 34 void main() 35 { 36 version(unittest) runUnitTests!mypackage(new ConsoleResultWriter); 37 else runApplication; 38 } 39 --- 40 */ 41 bool runUnitTests(alias composite)(TestResultWriter results) 42 { 43 assert(!g_runner, "Running multiple unit tests concurrently is not supported."); 44 g_runner = new TestRunner(results); 45 auto ret = g_runner.runUnitTests!composite(); 46 g_runner = null; 47 return ret; 48 } 49 50 51 /** 52 Emits a custom instrumentation value that will be stored in the unit test results. 53 */ 54 void instrument(string name, double value) 55 { 56 if (g_runner) g_runner.instrument(name, value); 57 } 58 59 60 /** 61 Attribute for giving a unit test a name. 62 63 The name of a unit test will be output in addition to the fully 64 qualified name of the test function in the test output. 65 66 Example: 67 --- 68 @name("some test") 69 unittest { 70 // unit test code 71 } 72 73 @name("some other test") 74 unittest { 75 // unit test code 76 } 77 --- 78 */ 79 struct name { string name; } 80 81 82 /** 83 Base interface for all unit test result writers. 84 */ 85 interface TestResultWriter { 86 void finalize(); 87 void beginTest(string name, string qualified_name); 88 void addScalar(Duration timestamp, string name, double value); 89 void endTest(Duration timestamp, Throwable error); 90 } 91 92 93 /** 94 Directly outputs unit test results to the console. 95 */ 96 class ConsoleTestResultWriter : TestResultWriter { 97 import std.stdio; 98 private { 99 size_t m_failCount, m_successCount; 100 string m_name, m_qualifiedName; 101 } 102 103 void finalize() 104 { 105 writefln("==========================="); 106 writefln("%s of %s tests have passed.", m_successCount, m_successCount+m_failCount); 107 writefln(""); 108 writefln("FINAL RESULT: %s", m_failCount > 0 ? "FAILED" : "PASSED"); 109 } 110 111 void beginTest(string name, string qualified_name) 112 { 113 m_name = name; 114 m_qualifiedName = qualified_name; 115 } 116 117 void addScalar(Duration timestamp, string name, double value) 118 { 119 } 120 121 void endTest(Duration timestamp, Throwable error) 122 { 123 if (error) { 124 writefln(`FAIL "%s" (%s) after %.6f s: %s`, m_name, m_qualifiedName, fracSecs(timestamp), error.msg); 125 m_failCount++; 126 } else { 127 writefln(`PASS "%s" (%s) after %.6f s`, m_name, m_qualifiedName, fracSecs(timestamp)); 128 m_successCount++; 129 } 130 } 131 132 static double fracSecs(Duration dur) 133 { 134 return 1E-6 * dur.total!"usecs"; 135 } 136 } 137 138 /** 139 Outputs test results and instrumentation values 140 */ 141 class JsonTestResultWriter : TestResultWriter { 142 import std.stdio; 143 144 private { 145 File m_file; 146 bool m_gotInstrumentValue, m_gotUnitTest; 147 } 148 149 this(string filename) 150 { 151 m_file = File(filename, "w+b"); 152 m_file.writeln("["); 153 m_gotUnitTest = false; 154 } 155 156 void finalize() 157 { 158 m_file.writeln(); 159 m_file.writeln("]"); 160 m_file.close(); 161 } 162 163 void beginTest(string name, string qualified_name) 164 { 165 if (m_gotUnitTest) m_file.writeln(","); 166 else m_gotUnitTest = true; 167 m_file.writef(`{"name": "%s", "qualifiedName": "%s", "instrumentation": [`, name, qualified_name); 168 m_gotInstrumentValue = false; 169 } 170 171 void addScalar(Duration timestamp, string name, double value) 172 { 173 if (m_gotInstrumentValue) m_file.write(", "); 174 else m_gotInstrumentValue = true; 175 m_file.writef(`{"name": "%s", "value": %s, "timestamp": %s}`, name, value, timestamp.total!"usecs" * 1E-6); 176 } 177 178 void endTest(Duration timestamp, Throwable error) 179 { 180 m_file.writef(`], "success": %s, "duration": %s`, error is null, timestamp.total!"usecs" * 1E-6); 181 if (error) m_file.writef(`, "message": "%s"`, error.msg); 182 m_file.write("}"); 183 } 184 } 185 186 187 private class TestRunner { 188 private { 189 Mutex m_mutex; 190 Condition m_condition; 191 TestResultWriter m_results; 192 InstrumentStats m_baseStats; 193 bool m_running, m_quit, m_instrumentsReady; 194 StopWatch m_stopWatch; 195 } 196 197 this(TestResultWriter writer) 198 { 199 m_results = writer; 200 m_mutex = new Mutex; 201 m_condition = new Condition(m_mutex); 202 } 203 204 bool runUnitTests(alias composite)() 205 { 206 InstrumentStats basestats; 207 m_running = false; 208 m_quit = false; 209 m_instrumentsReady = false; 210 211 auto instrumentthr = new Thread(&instrumentThread); 212 instrumentthr.name = "instrumentation thread"; 213 //instrumentthr.priority = Thread.PRIORITY_DEFAULT + 1; 214 instrumentthr.start(); 215 216 auto ret = runUnitTestsImpl!composite(); 217 m_results.finalize(); 218 219 synchronized (m_mutex) m_quit = true; 220 m_condition.notifyAll(); 221 instrumentthr.join(); 222 223 return ret; 224 } 225 226 private void instrument(string name, double value) 227 { 228 auto ts = cast(Duration)m_stopWatch.peek(); 229 synchronized (m_mutex) m_results.addScalar(ts, name, value); 230 } 231 232 private bool runUnitTestsImpl(alias composite)() 233 { 234 bool ret = true; 235 236 static if (composite.stringof.startsWith("module ") || isAggregateType!(typeof(composite))) { 237 foreach (test; __traits(getUnitTests, composite)) { 238 if (!runUnitTest!test()) 239 ret = false; 240 } 241 } 242 243 // if composite has members, descent recursively 244 static if (__traits(compiles, { auto mems = __traits(allMembers, composite); })) 245 foreach (M; __traits(allMembers, composite)) { 246 // stop on system types/modules and private members 247 static if (!isSystemModule!(composite, M)) 248 static if (__traits(compiles, { auto tup = TypeTuple!(__traits(getMember, composite, M)); })) 249 if (!runUnitTestsImpl!(__traits(getMember, composite, M))()) 250 ret = false; 251 } 252 253 return ret; 254 } 255 256 private bool runUnitTest(alias test)() 257 { 258 string name; 259 foreach (att; __traits(getAttributes, test)) 260 static if (is(typeof(att) == .name)) 261 name = att.name; 262 263 // wait for instrumentation thread to get ready 264 synchronized (m_mutex) 265 while (!m_instrumentsReady) 266 m_condition.wait(); 267 268 m_results.beginTest(name, fullyQualifiedName!test); 269 m_stopWatch.reset(); 270 271 GC.collect(); 272 m_baseStats = InstrumentStats.instrument(); 273 274 synchronized (m_mutex) m_running = true; 275 m_condition.notifyAll(); 276 277 Throwable error; 278 try { 279 m_stopWatch.start(); 280 test(); 281 m_stopWatch.stop(); 282 } catch (Throwable th) { 283 m_stopWatch.stop(); 284 error = th; 285 } 286 287 auto duration = cast(Duration)m_stopWatch.peek(); 288 289 auto stats = InstrumentStats.instrument(); 290 synchronized (m_mutex) { 291 writeInstrumentStats(m_results, duration, m_baseStats, stats); 292 m_running = false; 293 } 294 295 m_results.endTest(duration, error); 296 return error is null; 297 } 298 299 private void instrumentThread() 300 { 301 while (true) { 302 synchronized (m_mutex) { 303 if (m_quit) return; 304 if (!m_running) { 305 m_instrumentsReady = true; 306 m_condition.notifyAll(); 307 while (!m_running && !m_quit) m_condition.wait(); 308 if (m_quit) return; 309 m_instrumentsReady = false; 310 } 311 } 312 313 Thread.sleep(10.msecs); 314 315 auto ts = cast(Duration)m_stopWatch.peek; 316 auto stats = InstrumentStats.instrument(); 317 synchronized (m_mutex) 318 if (m_running) 319 writeInstrumentStats(m_results, ts, m_baseStats, stats); 320 } 321 } 322 } 323 324 325 // copied from gc.stats 326 private struct GCStats { 327 size_t poolsize; // total size of pool 328 size_t usedsize; // bytes allocated 329 size_t freeblocks; // number of blocks marked FREE 330 size_t freelistsize; // total of memory on free lists 331 size_t pageblocks; // number of blocks marked PAGE 332 } 333 334 // from gc.proxy 335 private extern (C) GCStats gc_stats(); 336 337 private struct InstrumentStats { 338 size_t gcPoolSize; 339 size_t gcUsedSize; 340 size_t gcFreeListSize; 341 342 static InstrumentStats instrument() 343 { 344 auto stats = gc_stats(); 345 InstrumentStats ret; 346 ret.gcPoolSize = stats.poolsize; 347 ret.gcUsedSize = stats.usedsize; 348 ret.gcFreeListSize = stats.freelistsize; 349 return ret; 350 } 351 } 352 353 private { 354 __gshared TestRunner g_runner; 355 } 356 357 private void writeInstrumentStats(TestResultWriter results, Duration timestamp, InstrumentStats base_stats, InstrumentStats stats) 358 { 359 results.addScalar(timestamp, "gcPoolSize", stats.gcPoolSize - base_stats.gcPoolSize); 360 results.addScalar(timestamp, "gcUsedSize", stats.gcUsedSize - base_stats.gcUsedSize); 361 results.addScalar(timestamp, "gcFreeListSize", stats.gcFreeListSize - base_stats.gcFreeListSize); 362 } 363 364 private bool isSystemModule(alias composite, string mem)() 365 { 366 return isSystemModule(fullyQualifiedName!(__traits(getMember, composite, mem))); 367 } 368 369 private bool isSystemModule()(string qualified_name) 370 { 371 return qualified_name.startsWith("std.") || 372 qualified_name.startsWith("core.") || 373 qualified_name.startsWith("object.") || 374 qualified_name == "object"; 375 }