1 /** 2 Unit test runner with structured result output support. 3 4 Copyright: Copyright ©2013 rejectedsoftware e.K., all rights reserved. 5 Authors: Sönke Ludwig 6 */ 7 module tested; 8 9 import core.sync.condition; 10 import core.sync.mutex; 11 import core.memory; 12 import core.time; 13 import core.thread; 14 import std.datetime : StopWatch; 15 import std.string : startsWith; 16 import std.traits : isAggregateType, fullyQualifiedName; 17 import std.typetuple : TypeTuple; 18 19 20 /** 21 Runs all unit tests contained in the given symbol recursively. 22 23 composite can be a package, a module or a composite type. 24 25 Example: 26 This example assumes that the application has all of its sources in 27 the package named "mypackage". All contained tests will be run 28 recursively. 29 30 --- 31 import tested; 32 import mypackage.application; 33 34 void main() 35 { 36 version(unittest) runUnitTests!mypackage(new ConsoleResultWriter); 37 else runApplication; 38 } 39 --- 40 */ 41 bool runUnitTests(alias composite)(TestResultWriter results) 42 { 43 assert(!g_runner, "Running multiple unit tests concurrently is not supported."); 44 g_runner = new TestRunner(results); 45 auto ret = g_runner.runUnitTests!composite(); 46 g_runner = null; 47 return ret; 48 } 49 50 51 /** 52 Emits a custom instrumentation value that will be stored in the unit test results. 53 */ 54 void instrument(string name, double value) 55 { 56 if (g_runner) g_runner.instrument(name, value); 57 } 58 59 60 /** 61 Attribute for giving a unit test a name. 62 63 The name of a unit test will be output in addition to the fully 64 qualified name of the test function in the test output. 65 66 Example: 67 --- 68 @name("some test") 69 unittest { 70 // unit test code 71 } 72 73 @name("some other test") 74 unittest { 75 // unit test code 76 } 77 --- 78 */ 79 struct name { string name; } 80 81 82 /** 83 Base interface for all unit test result writers. 84 */ 85 interface TestResultWriter { 86 void finalize(); 87 void beginTest(string name, string qualified_name); 88 void addScalar(Duration timestamp, string name, double value); 89 void endTest(Duration timestamp, Throwable error); 90 } 91 92 93 /** 94 Directly outputs unit test results to the console. 95 */ 96 class ConsoleTestResultWriter : TestResultWriter { 97 import std.stdio; 98 private { 99 size_t m_failCount, m_successCount; 100 string m_name, m_qualifiedName; 101 } 102 103 void finalize() 104 { 105 writefln("==========================="); 106 writefln("%s of %s tests have passed.", m_successCount, m_successCount+m_failCount); 107 writefln(""); 108 writefln("FINAL RESULT: %s", m_failCount > 0 ? "FAILED" : "PASSED"); 109 } 110 111 void beginTest(string name, string qualified_name) 112 { 113 m_name = name; 114 m_qualifiedName = qualified_name; 115 } 116 117 void addScalar(Duration timestamp, string name, double value) 118 { 119 } 120 121 void endTest(Duration timestamp, Throwable error) 122 { 123 if (error) { 124 writefln(`FAIL "%s" (%s) after %.6f s: %s`, m_name, m_qualifiedName, fracSecs(timestamp), error.msg); 125 m_failCount++; 126 } else { 127 writefln(`PASS "%s" (%s) after %.6f s`, m_name, m_qualifiedName, fracSecs(timestamp)); 128 m_successCount++; 129 } 130 } 131 132 static double fracSecs(Duration dur) 133 { 134 return 1E-6 * dur.total!"usecs"; 135 } 136 } 137 138 /** 139 Outputs test results and instrumentation values 140 */ 141 class JsonTestResultWriter : TestResultWriter { 142 import std.stdio; 143 144 private { 145 File m_file; 146 bool m_gotInstrumentValue, m_gotUnitTest; 147 } 148 149 this(string filename) 150 { 151 m_file = File(filename, "w+b"); 152 m_file.writeln("["); 153 m_gotUnitTest = false; 154 } 155 156 void finalize() 157 { 158 m_file.writeln(); 159 m_file.writeln("]"); 160 m_file.close(); 161 } 162 163 void beginTest(string name, string qualified_name) 164 { 165 if (m_gotUnitTest) m_file.writeln(","); 166 else m_gotUnitTest = true; 167 m_file.writef(`{"name": "%s", "qualifiedName": "%s", "instrumentation": [`, name, qualified_name); 168 m_gotInstrumentValue = false; 169 } 170 171 void addScalar(Duration timestamp, string name, double value) 172 { 173 if (m_gotInstrumentValue) m_file.write(", "); 174 else m_gotInstrumentValue = true; 175 m_file.writef(`{"name": "%s", "value": %s, "timestamp": %s}`, name, value, timestamp.total!"usecs" * 1E-6); 176 } 177 178 void endTest(Duration timestamp, Throwable error) 179 { 180 m_file.writef(`], "success": %s, "duration": %s`, error is null, timestamp.total!"usecs" * 1E-6); 181 if (error) m_file.writef(`, "message": "%s"`, error.msg); 182 m_file.write("}"); 183 } 184 } 185 186 187 private class TestRunner { 188 private { 189 Mutex m_mutex; 190 Condition m_condition; 191 TestResultWriter m_results; 192 InstrumentStats m_baseStats; 193 bool m_running, m_quit, m_instrumentsReady; 194 StopWatch m_stopWatch; 195 } 196 197 this(TestResultWriter writer) 198 { 199 m_results = writer; 200 m_mutex = new Mutex; 201 m_condition = new Condition(m_mutex); 202 } 203 204 bool runUnitTests(alias composite)() 205 { 206 InstrumentStats basestats; 207 m_running = false; 208 m_quit = false; 209 m_instrumentsReady = false; 210 211 auto instrumentthr = new Thread(&instrumentThread); 212 instrumentthr.name = "instrumentation thread"; 213 //instrumentthr.priority = Thread.PRIORITY_DEFAULT + 1; 214 instrumentthr.start(); 215 216 auto ret = runUnitTestsImpl!composite(); 217 m_results.finalize(); 218 219 synchronized (m_mutex) m_quit = true; 220 m_condition.notifyAll(); 221 instrumentthr.join(); 222 223 return ret; 224 } 225 226 private void instrument(string name, double value) 227 { 228 auto ts = cast(Duration)m_stopWatch.peek(); 229 synchronized (m_mutex) m_results.addScalar(ts, name, value); 230 } 231 232 private bool runUnitTestsImpl(alias composite)() 233 { 234 bool ret = true; 235 236 static if (composite.stringof.startsWith("module ") || isAggregateType!(typeof(composite))) { 237 foreach (test; __traits(getUnitTests, composite)) { 238 if (!runUnitTest!test()) 239 ret = false; 240 } 241 } 242 243 // if composite has members, descent recursively 244 static if (__traits(compiles, { auto mems = __traits(allMembers, composite); })) 245 foreach (M; __traits(allMembers, composite)) { 246 // stop on system types/modules and private members 247 static if (!isSystemModule!(composite, M)) 248 static if (__traits(compiles, { auto tup = TypeTuple!(__traits(getMember, composite, M)); })) 249 if (!runUnitTestsImpl!(__traits(getMember, composite, M))()) 250 ret = false; 251 } 252 253 return ret; 254 } 255 256 private bool runUnitTest(alias test)() 257 { 258 string name; 259 foreach (att; __traits(getAttributes, test)) 260 static if (is(typeof(att) == .name)) 261 name = att.name; 262 263 // wait for instrumentation thread to get ready 264 synchronized (m_mutex) 265 while (!m_instrumentsReady) 266 m_condition.wait(); 267 268 m_results.beginTest(name, fullyQualifiedName!test); 269 m_stopWatch.reset(); 270 271 GC.collect(); 272 m_baseStats = InstrumentStats.instrument(); 273 274 synchronized (m_mutex) m_running = true; 275 m_condition.notifyAll(); 276 277 Throwable error; 278 try { 279 m_stopWatch.start(); 280 test(); 281 m_stopWatch.stop(); 282 } catch (Throwable th) { 283 m_stopWatch.stop(); 284 error = th; 285 } 286 287 auto duration = cast(Duration)m_stopWatch.peek(); 288 289 auto stats = InstrumentStats.instrument(); 290 writeInstrumentStats(m_results, duration, m_baseStats, stats); 291 292 synchronized (m_mutex) m_running = false; 293 m_results.endTest(duration, error); 294 return error is null; 295 } 296 297 private void instrumentThread() 298 { 299 while (true) { 300 synchronized (m_mutex) { 301 if (m_quit) return; 302 if (!m_running) { 303 m_instrumentsReady = true; 304 m_condition.notifyAll(); 305 while (!m_running && !m_quit) m_condition.wait(); 306 if (m_quit) return; 307 m_instrumentsReady = false; 308 } 309 } 310 311 Thread.sleep(10.msecs); 312 313 auto ts = cast(Duration)m_stopWatch.peek; 314 auto stats = InstrumentStats.instrument(); 315 synchronized (m_mutex) 316 if (m_running) 317 writeInstrumentStats(m_results, ts, m_baseStats, stats); 318 } 319 } 320 } 321 322 323 // copied from gc.stats 324 private struct GCStats { 325 size_t poolsize; // total size of pool 326 size_t usedsize; // bytes allocated 327 size_t freeblocks; // number of blocks marked FREE 328 size_t freelistsize; // total of memory on free lists 329 size_t pageblocks; // number of blocks marked PAGE 330 } 331 332 // from gc.proxy 333 private extern (C) GCStats gc_stats(); 334 335 private struct InstrumentStats { 336 size_t gcPoolSize; 337 size_t gcUsedSize; 338 size_t gcFreeListSize; 339 340 static InstrumentStats instrument() 341 { 342 auto stats = gc_stats(); 343 InstrumentStats ret; 344 ret.gcPoolSize = stats.poolsize; 345 ret.gcUsedSize = stats.usedsize; 346 ret.gcFreeListSize = stats.freelistsize; 347 return ret; 348 } 349 } 350 351 private { 352 __gshared TestRunner g_runner; 353 } 354 355 private void writeInstrumentStats(TestResultWriter results, Duration timestamp, InstrumentStats base_stats, InstrumentStats stats) 356 { 357 results.addScalar(timestamp, "gcPoolSize", stats.gcPoolSize - base_stats.gcPoolSize); 358 results.addScalar(timestamp, "gcUsedSize", stats.gcUsedSize - base_stats.gcUsedSize); 359 results.addScalar(timestamp, "gcFreeListSize", stats.gcFreeListSize - base_stats.gcFreeListSize); 360 } 361 362 private bool isSystemModule(alias composite, string mem)() 363 { 364 return isSystemModule(fullyQualifiedName!(__traits(getMember, composite, mem))); 365 } 366 367 private bool isSystemModule()(string qualified_name) 368 { 369 return qualified_name.startsWith("std.") || 370 qualified_name.startsWith("core.") || 371 qualified_name.startsWith("object.") || 372 qualified_name == "object"; 373 }