Line data Source code
1 : // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
2 : // for details. All rights reserved. Use of this source code is governed by a
3 : // BSD-style license that can be found in the LICENSE file.
4 :
5 : import 'dart:async';
6 : import 'dart:collection';
7 :
8 : import 'package:async/async.dart' hide Result;
9 : import 'package:collection/collection.dart';
10 : import 'package:pool/pool.dart';
11 :
12 : import '../backend/group.dart';
13 : import '../backend/invoker.dart';
14 : import '../backend/live_test.dart';
15 : import '../backend/live_test_controller.dart';
16 : import '../backend/message.dart';
17 : import '../backend/state.dart';
18 : import '../backend/test.dart';
19 : import '../util/iterable_set.dart';
20 : import 'live_suite.dart';
21 : import 'live_suite_controller.dart';
22 : import 'load_suite.dart';
23 : import 'runner_suite.dart';
24 :
25 : /// An [Engine] manages a run that encompasses multiple test suites.
26 : ///
27 : /// Test suites are provided by passing them into [suiteSink]. Once all suites
28 : /// have been provided, the user should close [suiteSink] to indicate this.
29 : /// [run] won't terminate until [suiteSink] is closed. Suites will be run in the
30 : /// order they're provided to [suiteSink]. Tests within those suites will
31 : /// likewise be run in the order they're declared.
32 : ///
33 : /// The current status of every test is visible via [liveTests]. [onTestStarted]
34 : /// can also be used to be notified when a test is about to be run.
35 : ///
36 : /// The engine has some special logic for [LoadSuite]s and the tests they
37 : /// contain, referred to as "load tests". Load tests exist to provide visibility
38 : /// into the process of loading test files, but as long as that process is
39 : /// proceeding normally users usually don't care about it, so the engine only
40 : /// surfaces running load tests (that is, includes them in [liveTests] and other
41 : /// collections) under specific circumstances.
42 : ///
43 : /// If only load tests are running, exactly one load test will be in [active]
44 : /// and [liveTests]. If this test passes, it will be removed from both [active]
45 : /// and [liveTests] and *will not* be added to [passed]. If at any point a load
46 : /// test fails, it will be added to [failed] and [liveTests].
47 : ///
48 : /// The test suite loaded by a load suite will be automatically be run by the
49 : /// engine; it doesn't need to be added to [suiteSink] manually.
50 : ///
51 : /// Load tests will always be emitted through [onTestStarted] so users can watch
52 : /// their event streams once they start running.
53 : class Engine {
54 : /// Whether [run] has been called yet.
55 : var _runCalled = false;
56 :
57 : /// Whether [close] has been called.
58 : var _closed = false;
59 :
60 : /// Whether [close] was called before all the tests finished running.
61 : ///
62 : /// This is `null` if close hasn't been called and the tests are still
63 : /// running, `true` if close was called before the tests finished running, and
64 : /// `false` if the tests finished running before close was called.
65 : var _closedBeforeDone;
66 :
67 : /// A pool that limits the number of test suites running concurrently.
68 : final Pool _runPool;
69 :
70 : /// A pool that limits the number of test suites loaded concurrently.
71 : ///
72 : /// Once this reaches its limit, loading any additional test suites will cause
73 : /// previous suites to be unloaded in the order they completed.
74 : final Pool _loadPool;
75 :
76 : /// A completer that will complete when [this] is unpaused.
77 : ///
78 : /// If [this] isn't paused, [_pauseCompleter] is `null`.
79 : Completer _pauseCompleter;
80 :
81 : /// A future that completes once [this] is unpaused.
82 : ///
83 : /// If [this] isn't paused, this completes immediately.
84 : Future get _onUnpaused =>
85 10 : _pauseCompleter == null ? new Future.value() : _pauseCompleter.future;
86 :
87 : /// Whether all tests passed or were skipped.
88 : ///
89 : /// This fires once all tests have completed and [suiteSink] has been closed.
90 : /// This will be `null` if [close] was called before all the tests finished
91 : /// running.
92 : Future<bool> get success async {
93 5 : await Future
94 30 : .wait(<Future>[_group.future, _loadPool.done], eagerError: true);
95 5 : if (_closedBeforeDone) return null;
96 25 : return liveTests.every((liveTest) => liveTest.state.result.isPassing);
97 0 : }
98 :
99 : /// A group of futures for each test suite.
100 : final _group = new FutureGroup();
101 :
102 : /// All of the engine's stream subscriptions.
103 : final _subscriptions = new Set<StreamSubscription>();
104 :
105 : /// A sink used to pass [RunnerSuite]s in to the engine to run.
106 : ///
107 : /// Suites may be added as quickly as they're available; the Engine will only
108 : /// run as many as necessary at a time based on its concurrency settings.
109 : ///
110 : /// Suites added to the sink will be closed by the engine based on its
111 : /// internal logic.
112 15 : Sink<RunnerSuite> get suiteSink => new DelegatingSink(_suiteController.sink);
113 : final _suiteController = new StreamController<RunnerSuite>();
114 :
115 : /// All the [RunnerSuite]s added to [suiteSink] so far.
116 : ///
117 : /// Note that if a [LoadSuite] is added, this will only contain that suite,
118 : /// not the suite it loads.
119 0 : Set<RunnerSuite> get addedSuites => new UnmodifiableSetView(_addedSuites);
120 : final _addedSuites = new Set<RunnerSuite>();
121 :
122 : /// A broadcast stream that emits each [RunnerSuite] as it's added to the
123 : /// engine via [suiteSink].
124 : ///
125 : /// Note that if a [LoadSuite] is added, this will only return that suite, not
126 : /// the suite it loads.
127 : ///
128 : /// This is guaranteed to fire after the suite is added to [addedSuites].
129 0 : Stream<RunnerSuite> get onSuiteAdded => _onSuiteAddedController.stream;
130 : final _onSuiteAddedController = new StreamController<RunnerSuite>.broadcast();
131 :
132 : /// All the currently-known suites that have run or are running.
133 : ///
134 : /// These are [LiveSuite]s, representing the in-progress state of each suite
135 : /// as its component tests are being run.
136 : ///
137 : /// Note that unlike [addedSuites], for suites that are loaded using
138 : /// [LoadSuite]s, both the [LoadSuite] and the suite it loads will eventually
139 : /// be in this set.
140 0 : Set<LiveSuite> get liveSuites => new UnmodifiableSetView(_liveSuites);
141 : final _liveSuites = new Set<LiveSuite>();
142 :
143 : /// A broadcast stream that emits each [LiveSuite] as it's loaded.
144 : ///
145 : /// Note that unlike [onSuiteAdded], for suites that are loaded using
146 : /// [LoadSuite]s, both the [LoadSuite] and the suite it loads will eventually
147 : /// be emitted by this stream.
148 0 : Stream<LiveSuite> get onSuiteStarted => _onSuiteStartedController.stream;
149 : final _onSuiteStartedController = new StreamController<LiveSuite>.broadcast();
150 :
151 : /// All the currently-known tests that have run or are running.
152 : ///
153 : /// These are [LiveTest]s, representing the in-progress state of each test.
154 : /// Tests that have not yet begun running are marked [Status.pending]; tests
155 : /// that have finished are marked [Status.complete].
156 : ///
157 : /// This is guaranteed to contain the same tests as the union of [passed],
158 : /// [skipped], [failed], and [active].
159 : ///
160 : /// [LiveTest.run] must not be called on these tests.
161 : Set<LiveTest> get liveTests =>
162 35 : new UnionSet.from([passed, skipped, failed, new IterableSet(active)],
163 : disjoint: true);
164 :
165 : /// A stream that emits each [LiveTest] as it's about to start running.
166 : ///
167 : /// This is guaranteed to fire before [LiveTest.onStateChange] first fires.
168 10 : Stream<LiveTest> get onTestStarted => _onTestStartedGroup.stream;
169 : final _onTestStartedGroup = new StreamGroup<LiveTest>.broadcast();
170 :
171 : /// The set of tests that have completed and been marked as passing.
172 10 : Set<LiveTest> get passed => _passedGroup.set;
173 : final _passedGroup = new UnionSetController<LiveTest>(disjoint: true);
174 :
175 : /// The set of tests that have completed and been marked as skipped.
176 10 : Set<LiveTest> get skipped => _skippedGroup.set;
177 : final _skippedGroup = new UnionSetController<LiveTest>(disjoint: true);
178 :
179 : /// The set of tests that have completed and been marked as failing or error.
180 10 : Set<LiveTest> get failed => _failedGroup.set;
181 : final _failedGroup = new UnionSetController<LiveTest>(disjoint: true);
182 :
183 : /// The tests that are still running, in the order they begain running.
184 10 : List<LiveTest> get active => new UnmodifiableListView(_active);
185 : final _active = new QueueList<LiveTest>();
186 :
187 : /// The set of tests that have been marked for restarting.
188 : ///
189 : /// This is always a subset of [active]. Once a test in here has finished
190 : /// running, it's run again.
191 : final _restarted = new Set<LiveTest>();
192 :
193 : /// The tests from [LoadSuite]s that are still running, in the order they
194 : /// began running.
195 : ///
196 : /// This is separate from [active] because load tests aren't always surfaced.
197 : final _activeLoadTests = new List<LiveTest>();
198 :
199 : /// Whether this engine is idle—that is, not currently executing a test.
200 0 : bool get isIdle => _group.isIdle;
201 :
202 : /// A broadcast stream that fires an event whenever [isIdle] switches from
203 : /// `false` to `true`.
204 0 : Stream get onIdle => _group.onIdle;
205 :
206 : // TODO(nweiz): Use interface libraries to take a Configuration even when
207 : // dart:io is unavailable.
208 : /// Creates an [Engine] that will run all tests provided via [suiteSink].
209 : ///
210 : /// [concurrency] controls how many suites are run at once, and defaults to 1.
211 : /// [maxSuites] controls how many suites are *loaded* at once, and defaults to
212 : /// four times [concurrency].
213 : Engine({int concurrency, int maxSuites})
214 5 : : _runPool = new Pool(concurrency ?? 1),
215 15 : _loadPool = new Pool(maxSuites ?? (concurrency ?? 1) * 2) {
216 15 : _group.future.then((_) {
217 10 : _onTestStartedGroup.close();
218 10 : _onSuiteStartedController.close();
219 10 : if (_closedBeforeDone == null) _closedBeforeDone = false;
220 5 : }).catchError((_) {
221 : // Don't top-level errors. They'll be thrown via [success] anyway.
222 : });
223 : }
224 :
225 : /// Creates an [Engine] that will run all tests in [suites].
226 : ///
227 : /// An engine constructed this way will automatically close its [suiteSink],
228 : /// meaning that no further suites may be provided.
229 : ///
230 : /// [concurrency] controls how many suites are run at once. If [runSkipped] is
231 : /// `true`, skipped tests will be run as though they weren't skipped.
232 : factory Engine.withSuites(List<RunnerSuite> suites, {int concurrency}) {
233 0 : var engine = new Engine(concurrency: concurrency);
234 0 : for (var suite in suites) engine.suiteSink.add(suite);
235 0 : engine.suiteSink.close();
236 : return engine;
237 : }
238 :
239 : /// Runs all tests in all suites defined by this engine.
240 : ///
241 : /// This returns `true` if all tests succeed, and `false` otherwise. It will
242 : /// only return once all tests have finished running and [suiteSink] has been
243 : /// closed.
244 : Future<bool> run() {
245 5 : if (_runCalled) {
246 0 : throw new StateError("Engine.run() may not be called more than once.");
247 : }
248 5 : _runCalled = true;
249 :
250 : StreamSubscription subscription;
251 15 : subscription = _suiteController.stream.listen((suite) {
252 10 : _addedSuites.add(suite);
253 10 : _onSuiteAddedController.add(suite);
254 :
255 15 : _group.add(new Future.sync(() async {
256 15 : var loadResource = await _loadPool.request();
257 :
258 : var controller;
259 5 : if (suite is LoadSuite) {
260 0 : await _onUnpaused;
261 0 : controller = await _addLoadSuite(suite);
262 : if (controller == null) {
263 0 : loadResource.release();
264 : return;
265 : }
266 : } else {
267 5 : controller = new LiveSuiteController(suite);
268 : }
269 :
270 10 : _addLiveSuite(controller.liveSuite);
271 :
272 15 : await _runPool.withResource(() async {
273 5 : if (_closed) return;
274 30 : await _runGroup(controller, controller.liveSuite.suite.group, []);
275 5 : controller.noMoreLiveTests();
276 10 : loadResource.allowRelease(() => controller.close());
277 0 : });
278 0 : }));
279 : }, onDone: () {
280 10 : _subscriptions.remove(subscription);
281 10 : _onSuiteAddedController.close();
282 10 : _group.close();
283 10 : _loadPool.close();
284 : });
285 10 : _subscriptions.add(subscription);
286 :
287 5 : return success;
288 : }
289 :
290 : /// Runs all the entries in [group] in sequence.
291 : ///
292 : /// [suiteController] is the controller fo the suite that contains [group].
293 : /// [parents] is a list of groups that contain [group]. It may be modified,
294 : /// but it's guaranteed to be in its original state once this function has
295 : /// finished.
296 : Future _runGroup(LiveSuiteController suiteController, Group group,
297 : List<Group> parents) async {
298 5 : parents.add(group);
299 : try {
300 15 : var suiteConfig = suiteController.liveSuite.suite.config;
301 15 : var skipGroup = !suiteConfig.runSkipped && group.metadata.skip;
302 : var setUpAllSucceeded = true;
303 5 : if (!skipGroup && group.setUpAll != null) {
304 0 : var liveTest = group.setUpAll
305 0 : .load(suiteController.liveSuite.suite, groups: parents);
306 0 : await _runLiveTest(suiteController, liveTest, countSuccess: false);
307 0 : setUpAllSucceeded = liveTest.state.result.isPassing;
308 : }
309 :
310 5 : if (!_closed && setUpAllSucceeded) {
311 15 : for (var entry in group.entries) {
312 5 : if (_closed) return;
313 :
314 5 : if (entry is Group) {
315 10 : await _runGroup(suiteController, entry, parents);
316 15 : } else if (!suiteConfig.runSkipped && entry.metadata.skip) {
317 0 : await _runSkippedTest(suiteController, entry, parents);
318 : } else {
319 5 : var test = entry as Test;
320 10 : await _runLiveTest(suiteController,
321 15 : test.load(suiteController.liveSuite.suite, groups: parents));
322 : }
323 : }
324 : }
325 :
326 : // Even if we're closed or setUpAll failed, we want to run all the
327 : // teardowns to ensure that any state is properly cleaned up.
328 5 : if (!skipGroup && group.tearDownAll != null) {
329 0 : var liveTest = group.tearDownAll
330 0 : .load(suiteController.liveSuite.suite, groups: parents);
331 0 : await _runLiveTest(suiteController, liveTest, countSuccess: false);
332 0 : if (_closed) await liveTest.close();
333 : }
334 : } finally {
335 5 : parents.remove(group);
336 : }
337 0 : }
338 :
339 : /// Runs [liveTest] using [suiteController].
340 : ///
341 : /// If [countSuccess] is `true` (the default), the test is put into [passed]
342 : /// if it succeeds. Otherwise, it's removed from [liveTests] entirely.
343 : Future _runLiveTest(LiveSuiteController suiteController, LiveTest liveTest,
344 : {bool countSuccess: true}) async {
345 10 : await _onUnpaused;
346 10 : _active.add(liveTest);
347 :
348 : // If there were no active non-load tests, the current active test would
349 : // have been a load test. In that case, remove it, since now we have a
350 : // non-load test to add.
351 20 : if (_active.first.suite is LoadSuite) _active.removeFirst();
352 :
353 : StreamSubscription subscription;
354 10 : subscription = liveTest.onStateChange.listen((state) {
355 10 : if (state.status != Status.complete) return;
356 10 : _active.remove(liveTest);
357 :
358 : // If we're out of non-load tests, surface a load test.
359 20 : if (_active.isEmpty && _activeLoadTests.isNotEmpty) {
360 0 : _active.add(_activeLoadTests.first);
361 : }
362 : }, onDone: () {
363 0 : _subscriptions.remove(subscription);
364 : });
365 10 : _subscriptions.add(subscription);
366 :
367 5 : suiteController.reportLiveTest(liveTest, countSuccess: countSuccess);
368 :
369 : // Schedule a microtask to ensure that [onTestStarted] fires before the
370 : // first [LiveTest.onStateChange] event.
371 15 : await new Future.microtask(liveTest.run);
372 :
373 : // Once the test finishes, use [new Future] to do a coarse-grained event
374 : // loop pump to avoid starving non-microtask events.
375 10 : await new Future(() {});
376 :
377 10 : if (!_restarted.contains(liveTest)) return;
378 0 : await _runLiveTest(suiteController, liveTest.copy(),
379 : countSuccess: countSuccess);
380 0 : _restarted.remove(liveTest);
381 0 : }
382 :
383 : /// Runs a dummy [LiveTest] for a test marked as "skip".
384 : ///
385 : /// [suiteController] is the controller for the suite that contains [test].
386 : /// [parents] is a list of groups that contain [test].
387 : Future _runSkippedTest(LiveSuiteController suiteController, Test test,
388 : List<Group> parents) async {
389 0 : await _onUnpaused;
390 : var skipped =
391 0 : new LocalTest(test.name, test.metadata, () {}, trace: test.trace);
392 :
393 : var controller;
394 : controller =
395 0 : new LiveTestController(suiteController.liveSuite.suite, skipped, () {
396 0 : controller.setState(const State(Status.running, Result.success));
397 0 : controller.setState(const State(Status.running, Result.skipped));
398 :
399 0 : if (skipped.metadata.skipReason != null) {
400 : controller
401 0 : .message(new Message.skip("Skip: ${skipped.metadata.skipReason}"));
402 : }
403 :
404 0 : controller.setState(const State(Status.complete, Result.skipped));
405 0 : controller.completer.complete();
406 : }, () {}, groups: parents);
407 :
408 0 : return await _runLiveTest(suiteController, controller.liveTest);
409 0 : }
410 :
411 : /// Closes [liveTest] and tells the engine to re-run it once it's done
412 : /// running.
413 : ///
414 : /// Returns the same future as [LiveTest.close].
415 : Future restartTest(LiveTest liveTest) async {
416 0 : if (_activeLoadTests.contains(liveTest)) {
417 0 : throw new ArgumentError("Can't restart a load test.");
418 : }
419 :
420 0 : if (!_active.contains(liveTest)) {
421 0 : throw new StateError("Can't restart inactive test "
422 0 : "\"${liveTest.test.name}\".");
423 : }
424 :
425 0 : _restarted.add(liveTest);
426 0 : _active.remove(liveTest);
427 0 : await liveTest.close();
428 0 : }
429 :
430 : /// Runs [suite] and returns the [LiveSuiteController] for the suite it loads.
431 : ///
432 : /// Returns `null` if the suite fails to load.
433 : Future<LiveSuiteController> _addLoadSuite(LoadSuite suite) async {
434 0 : var controller = new LiveSuiteController(suite);
435 0 : _addLiveSuite(controller.liveSuite);
436 :
437 0 : var liveTest = suite.test.load(suite);
438 0 : _activeLoadTests.add(liveTest);
439 :
440 : // Only surface the load test if there are no other tests currently running.
441 0 : if (_active.isEmpty) _active.add(liveTest);
442 :
443 : StreamSubscription subscription;
444 0 : subscription = liveTest.onStateChange.listen((state) {
445 0 : if (state.status != Status.complete) return;
446 0 : _activeLoadTests.remove(liveTest);
447 :
448 : // Only one load test will be active at any given time, and it will always
449 : // be the only active test. Remove it and, if possible, surface another
450 : // load test.
451 0 : if (_active.isNotEmpty && _active.first.suite == suite) {
452 0 : _active.remove(liveTest);
453 0 : if (_activeLoadTests.isNotEmpty) _active.add(_activeLoadTests.last);
454 : }
455 : }, onDone: () {
456 0 : _subscriptions.remove(subscription);
457 : });
458 0 : _subscriptions.add(subscription);
459 :
460 0 : controller.reportLiveTest(liveTest, countSuccess: false);
461 0 : controller.noMoreLiveTests();
462 :
463 : // Schedule a microtask to ensure that [onTestStarted] fires before the
464 : // first [LiveTest.onStateChange] event.
465 0 : new Future.microtask(liveTest.run);
466 :
467 0 : var innerSuite = await suite.suite;
468 : if (innerSuite == null) return null;
469 :
470 0 : var innerController = new LiveSuiteController(innerSuite);
471 0 : innerController.liveSuite.onClose.then((_) {
472 : // When the main suite is closed, close the load suite and its test as
473 : // well. This doesn't release any resources, but it does close streams
474 : // which indicates that the load test won't experience an error in the
475 : // future.
476 0 : liveTest.close();
477 0 : controller.close();
478 : });
479 :
480 : return innerController;
481 0 : }
482 :
483 : /// Add [liveSuite] and the information it exposes to the engine's
484 : /// informational streams and collections.
485 : void _addLiveSuite(LiveSuite liveSuite) {
486 10 : _liveSuites.add(liveSuite);
487 10 : _onSuiteStartedController.add(liveSuite);
488 :
489 15 : _onTestStartedGroup.add(liveSuite.onTestStarted);
490 15 : _passedGroup.add(liveSuite.passed);
491 15 : _skippedGroup.add(liveSuite.skipped);
492 15 : _failedGroup.add(liveSuite.failed);
493 : }
494 :
495 : /// Pauses the engine.
496 : ///
497 : /// This pauses all streams and keeps any new suites from being loaded or
498 : /// tests from being run until [resume] is called.
499 : ///
500 : /// This does nothing if the engine is already paused. Pauses are *not*
501 : /// cumulative.
502 : void pause() {
503 0 : if (_pauseCompleter != null) return;
504 0 : _pauseCompleter = new Completer();
505 0 : for (var subscription in _subscriptions) {
506 0 : subscription.pause();
507 : }
508 : }
509 :
510 : void resume() {
511 0 : if (_pauseCompleter == null) return;
512 0 : _pauseCompleter.complete();
513 0 : _pauseCompleter = null;
514 0 : for (var subscription in _subscriptions) {
515 0 : subscription.resume();
516 : }
517 : }
518 :
519 : /// Signals that the caller is done paying attention to test results and the
520 : /// engine should release any resources it has allocated.
521 : ///
522 : /// Any actively-running tests are also closed. VM tests are allowed to finish
523 : /// running so that any modifications they've made to the filesystem can be
524 : /// cleaned up.
525 : ///
526 : /// **Note that closing the engine is not the same as closing [suiteSink].**
527 : /// Closing [suiteSink] indicates that no more input will be provided, closing
528 : /// the engine indicates that no more output should be emitted.
529 : Future close() async {
530 0 : _closed = true;
531 0 : if (_closedBeforeDone != null) _closedBeforeDone = true;
532 0 : _onSuiteAddedController.close();
533 0 : _suiteController.close();
534 :
535 : // Close the running tests first so that we're sure to wait for them to
536 : // finish before we close their suites and cause them to become unloaded.
537 0 : var allLiveTests = liveTests.toSet()..addAll(_activeLoadTests);
538 0 : var futures = allLiveTests.map((liveTest) => liveTest.close()).toList();
539 :
540 : // Closing the load pool will close the test suites as soon as their tests
541 : // are done. For browser suites this is effectively immediate since their
542 : // tests shut down as soon as they're closed, but for VM suites we may need
543 : // to wait for tearDowns or tearDownAlls to run.
544 0 : futures.add(_loadPool.close());
545 0 : await Future.wait(futures, eagerError: true);
546 0 : }
547 : }
|