Line data Source code
1 : // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
2 : // for details. All rights reserved. Use of this source code is governed by a
3 : // BSD-style license that can be found in the LICENSE file.
4 :
5 : import 'dart:async';
6 : import 'dart:math';
7 :
8 : import 'package:async/async.dart' hide Result;
9 : import 'package:collection/collection.dart';
10 : import 'package:pool/pool.dart';
11 : import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
12 : import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
13 : import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports
14 : import 'package:test_api/src/backend/live_test_controller.dart'; // ignore: implementation_imports
15 : import 'package:test_api/src/backend/message.dart'; // ignore: implementation_imports
16 : import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports
17 : import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
18 :
19 : import 'coverage_stub.dart' if (dart.library.io) 'coverage.dart';
20 : import 'live_suite.dart';
21 : import 'live_suite_controller.dart';
22 : import 'load_suite.dart';
23 : import 'runner_suite.dart';
24 : import 'util/iterable_set.dart';
25 :
26 : /// An [Engine] manages a run that encompasses multiple test suites.
27 : ///
28 : /// Test suites are provided by passing them into [suiteSink]. Once all suites
29 : /// have been provided, the user should close [suiteSink] to indicate this.
30 : /// [run] won't terminate until [suiteSink] is closed. Suites will be run in the
31 : /// order they're provided to [suiteSink]. Tests within those suites will
32 : /// likewise be run in the order they're declared.
33 : ///
34 : /// The current status of every test is visible via [liveTests]. [onTestStarted]
35 : /// can also be used to be notified when a test is about to be run.
36 : ///
37 : /// The engine has some special logic for [LoadSuite]s and the tests they
38 : /// contain, referred to as "load tests". Load tests exist to provide visibility
39 : /// into the process of loading test files. As long as that process is
40 : /// proceeding normally users usually don't care about it, so the engine does
41 : /// not include them in [liveTests] and other collections.
42 : /// If a load test fails, it will be added to [failed] and [liveTests].
43 : ///
44 : /// The test suite loaded by a load suite will be automatically be run by the
45 : /// engine; it doesn't need to be added to [suiteSink] manually.
46 : ///
47 : /// Load tests will always be emitted through [onTestStarted] so users can watch
48 : /// their event streams once they start running.
49 : class Engine {
50 : /// Whether [run] has been called yet.
51 : var _runCalled = false;
52 :
53 : /// Whether [close] has been called.
54 : var _closed = false;
55 :
56 : /// Whether [close] was called before all the tests finished running.
57 : ///
58 : /// This is `null` if close hasn't been called and the tests are still
59 : /// running, `true` if close was called before the tests finished running, and
60 : /// `false` if the tests finished running before close was called.
61 : bool? _closedBeforeDone;
62 :
63 : /// The coverage output directory.
64 : String? _coverage;
65 :
66 : /// The seed used to generate randomess for test case shuffling.
67 : ///
68 : /// If null or zero no shuffling will occur.
69 : /// The same seed will shuffle the tests in the same way every time.
70 : int? testRandomizeOrderingSeed;
71 :
72 : /// A pool that limits the number of test suites running concurrently.
73 : final Pool _runPool;
74 :
75 : /// A completer that will complete when this engine is unpaused.
76 : ///
77 : /// `null` if this engine is not paused.
78 : Completer? _pauseCompleter;
79 :
80 : /// A future that completes once this is unpaused.
81 : ///
82 : /// If this engine isn't paused, this future completes immediately.
83 0 : Future get _onUnpaused =>
84 0 : _pauseCompleter == null ? Future.value() : _pauseCompleter!.future;
85 :
86 : /// Whether all tests passed or were skipped.
87 : ///
88 : /// This fires once all tests have completed and [suiteSink] has been closed.
89 : /// This will be `null` if [close] was called before all the tests finished
90 : /// running.
91 0 : Future<bool?> get success async {
92 0 : await Future.wait(<Future>[_group.future, _runPool.done], eagerError: true);
93 0 : if (_closedBeforeDone!) return null;
94 0 : return liveTests.every((liveTest) =>
95 0 : liveTest.state.result.isPassing &&
96 0 : liveTest.state.status == Status.complete);
97 : }
98 :
99 : /// A group of futures for each test suite.
100 : final _group = FutureGroup();
101 :
102 : /// All of the engine's stream subscriptions.
103 : final _subscriptions = <StreamSubscription>{};
104 :
105 : /// A sink used to pass [RunnerSuite]s in to the engine to run.
106 : ///
107 : /// Suites may be added as quickly as they're available; the Engine will only
108 : /// run as many as necessary at a time based on its concurrency settings.
109 : ///
110 : /// Suites added to the sink will be closed by the engine based on its
111 : /// internal logic.
112 0 : Sink<RunnerSuite> get suiteSink => DelegatingSink(_suiteController.sink);
113 : final _suiteController = StreamController<RunnerSuite>();
114 :
115 : /// All the [RunnerSuite]s added to [suiteSink] so far.
116 : ///
117 : /// Note that if a [LoadSuite] is added, this will only contain that suite,
118 : /// not the suite it loads.
119 0 : Set<RunnerSuite> get addedSuites => UnmodifiableSetView(_addedSuites);
120 : final _addedSuites = <RunnerSuite>{};
121 :
122 : /// A broadcast stream that emits each [RunnerSuite] as it's added to the
123 : /// engine via [suiteSink].
124 : ///
125 : /// Note that if a [LoadSuite] is added, this will only return that suite, not
126 : /// the suite it loads.
127 : ///
128 : /// This is guaranteed to fire after the suite is added to [addedSuites].
129 0 : Stream<RunnerSuite> get onSuiteAdded => _onSuiteAddedController.stream;
130 : final _onSuiteAddedController = StreamController<RunnerSuite>.broadcast();
131 :
132 : /// A broadcast stream that emits each [LiveSuite] as it's loaded.
133 : ///
134 : /// Note that unlike [onSuiteAdded], for suites that are loaded using
135 : /// [LoadSuite]s, both the [LoadSuite] and the suite it loads will eventually
136 : /// be emitted by this stream.
137 0 : Stream<LiveSuite> get onSuiteStarted => _onSuiteStartedController.stream;
138 : final _onSuiteStartedController = StreamController<LiveSuite>.broadcast();
139 :
140 : /// All the currently-known tests that have run or are running.
141 : ///
142 : /// These are [LiveTest]s, representing the in-progress state of each test.
143 : /// Tests that have not yet begun running are marked [Status.pending]; tests
144 : /// that have finished are marked [Status.complete].
145 : ///
146 : /// This is guaranteed to contain the same tests as the union of [passed],
147 : /// [skipped], [failed], and [active].
148 : ///
149 : /// [LiveTest.run] must not be called on these tests.
150 0 : Set<LiveTest> get liveTests =>
151 0 : UnionSet.from([passed, skipped, failed, IterableSet(active)],
152 : disjoint: true);
153 :
154 : /// A stream that emits each [LiveTest] as it's about to start running.
155 : ///
156 : /// This is guaranteed to fire before [LiveTest.onStateChange] first fires.
157 0 : Stream<LiveTest> get onTestStarted => _onTestStartedGroup.stream;
158 : final _onTestStartedGroup = StreamGroup<LiveTest>.broadcast();
159 :
160 : /// The set of tests that have completed and been marked as passing.
161 0 : Set<LiveTest> get passed => _passedGroup.set;
162 : final _passedGroup = UnionSetController<LiveTest>(disjoint: true);
163 :
164 : /// The set of tests that have completed and been marked as skipped.
165 0 : Set<LiveTest> get skipped => _skippedGroup.set;
166 : final _skippedGroup = UnionSetController<LiveTest>(disjoint: true);
167 :
168 : /// The set of tests that have completed and been marked as failing or error.
169 0 : Set<LiveTest> get failed => _failedGroup.set;
170 : final _failedGroup = UnionSetController<LiveTest>(disjoint: true);
171 :
172 : /// The tests that are still running, in the order they began running.
173 0 : List<LiveTest> get active => UnmodifiableListView(_active);
174 : final _active = QueueList<LiveTest>();
175 :
176 : /// The suites that are still loading, in the order they began.
177 0 : List<LiveTest> get activeSuiteLoads =>
178 0 : UnmodifiableListView(_activeSuiteLoads);
179 : final _activeSuiteLoads = <LiveTest>{};
180 :
181 : /// The set of tests that have been marked for restarting.
182 : ///
183 : /// This is always a subset of [active]. Once a test in here has finished
184 : /// running, it's run again.
185 : final _restarted = <LiveTest>{};
186 :
187 : /// Whether this engine is idle—that is, not currently executing a test.
188 0 : bool get isIdle => _group.isIdle;
189 :
190 : /// A broadcast stream that fires an event whenever [isIdle] switches from
191 : /// `false` to `true`.
192 0 : Stream get onIdle => _group.onIdle;
193 :
194 : /// Creates an [Engine] that will run all tests provided via [suiteSink].
195 : ///
196 : /// [concurrency] controls how many suites are loaded and ran at once, and
197 : /// defaults to 1.
198 : ///
199 : /// [testRandomizeOrderingSeed] configures test case shuffling within each
200 : /// test suite.
201 : /// Any non-zero value will enable shuffling using this value as a seed.
202 : /// Omitting this argument or passing `0` disables shuffling.
203 : ///
204 : /// [coverage] specifies a directory to output coverage information.
205 0 : Engine({int? concurrency, String? coverage, this.testRandomizeOrderingSeed})
206 0 : : _runPool = Pool(concurrency ?? 1),
207 : _coverage = coverage {
208 0 : _group.future.then((_) {
209 0 : _onTestStartedGroup.close();
210 0 : _onSuiteStartedController.close();
211 0 : _closedBeforeDone ??= false;
212 0 : }).onError((_, __) {
213 : // Don't top-level errors. They'll be thrown via [success] anyway.
214 : });
215 : }
216 :
217 : /// Creates an [Engine] that will run all tests in [suites].
218 : ///
219 : /// An engine constructed this way will automatically close its [suiteSink],
220 : /// meaning that no further suites may be provided.
221 : ///
222 : /// [concurrency] controls how many suites are run at once. If [runSkipped] is
223 : /// `true`, skipped tests will be run as though they weren't skipped.
224 0 : factory Engine.withSuites(List<RunnerSuite> suites,
225 : {int? concurrency, String? coverage}) {
226 0 : var engine = Engine(concurrency: concurrency, coverage: coverage);
227 0 : for (var suite in suites) {
228 0 : engine.suiteSink.add(suite);
229 : }
230 0 : engine.suiteSink.close();
231 : return engine;
232 : }
233 :
234 : /// Runs all tests in all suites defined by this engine.
235 : ///
236 : /// This returns `true` if all tests succeed, and `false` otherwise. It will
237 : /// only return once all tests have finished running and [suiteSink] has been
238 : /// closed.
239 : ///
240 : /// If [success] completes with `null` this will complete with `null`.
241 0 : Future<bool?> run() {
242 0 : if (_runCalled) {
243 0 : throw StateError('Engine.run() may not be called more than once.');
244 : }
245 0 : _runCalled = true;
246 :
247 0 : var subscription = _suiteController.stream.listen(null);
248 : subscription
249 0 : ..onData((suite) {
250 0 : _addedSuites.add(suite);
251 0 : _onSuiteAddedController.add(suite);
252 :
253 0 : _group.add(() async {
254 0 : var resource = await _runPool.request();
255 : LiveSuiteController? controller;
256 : try {
257 0 : if (suite is LoadSuite) {
258 0 : await _onUnpaused;
259 0 : controller = await _addLoadSuite(suite);
260 : if (controller == null) return;
261 : } else {
262 0 : controller = LiveSuiteController(suite);
263 : }
264 :
265 0 : _addLiveSuite(controller.liveSuite);
266 :
267 0 : if (_closed) return;
268 0 : await _runGroup(controller, controller.liveSuite.suite.group, []);
269 0 : controller.noMoreLiveTests();
270 0 : if (_coverage != null) await writeCoverage(_coverage!, controller);
271 : } finally {
272 0 : resource.allowRelease(() => controller?.close());
273 : }
274 : }());
275 : })
276 0 : ..onDone(() {
277 0 : _subscriptions.remove(subscription);
278 0 : _onSuiteAddedController.close();
279 0 : _group.close();
280 0 : _runPool.close();
281 : });
282 0 : _subscriptions.add(subscription);
283 :
284 0 : return success;
285 : }
286 :
287 : /// Runs all the entries in [group] in sequence.
288 : ///
289 : /// [suiteController] is the controller fo the suite that contains [group].
290 : /// [parents] is a list of groups that contain [group]. It may be modified,
291 : /// but it's guaranteed to be in its original state once this function has
292 : /// finished.
293 0 : Future _runGroup(LiveSuiteController suiteController, Group group,
294 : List<Group> parents) async {
295 0 : parents.add(group);
296 : try {
297 0 : var suiteConfig = suiteController.liveSuite.suite.config;
298 0 : var skipGroup = !suiteConfig.runSkipped && group.metadata.skip;
299 : var setUpAllSucceeded = true;
300 0 : if (!skipGroup && group.setUpAll != null) {
301 0 : var liveTest = group.setUpAll!
302 0 : .load(suiteController.liveSuite.suite, groups: parents);
303 0 : await _runLiveTest(suiteController, liveTest, countSuccess: false);
304 0 : setUpAllSucceeded = liveTest.state.result.isPassing;
305 : }
306 :
307 0 : if (!_closed && setUpAllSucceeded) {
308 : // shuffle the group entries
309 0 : var entries = group.entries.toList();
310 0 : if (suiteConfig.allowTestRandomization &&
311 0 : testRandomizeOrderingSeed != null &&
312 0 : testRandomizeOrderingSeed! > 0) {
313 0 : entries.shuffle(Random(testRandomizeOrderingSeed));
314 : }
315 :
316 0 : for (var entry in entries) {
317 0 : if (_closed) return;
318 :
319 0 : if (entry is Group) {
320 0 : await _runGroup(suiteController, entry, parents);
321 0 : } else if (!suiteConfig.runSkipped && entry.metadata.skip) {
322 0 : await _runSkippedTest(suiteController, entry as Test, parents);
323 : } else {
324 : var test = entry as Test;
325 0 : await _runLiveTest(suiteController,
326 0 : test.load(suiteController.liveSuite.suite, groups: parents));
327 : }
328 : }
329 : }
330 :
331 : // Even if we're closed or setUpAll failed, we want to run all the
332 : // teardowns to ensure that any state is properly cleaned up.
333 0 : if (!skipGroup && group.tearDownAll != null) {
334 0 : var liveTest = group.tearDownAll!
335 0 : .load(suiteController.liveSuite.suite, groups: parents);
336 0 : await _runLiveTest(suiteController, liveTest, countSuccess: false);
337 0 : if (_closed) await liveTest.close();
338 : }
339 : } finally {
340 0 : parents.remove(group);
341 : }
342 : }
343 :
344 : /// Runs [liveTest] using [suiteController].
345 : ///
346 : /// If [countSuccess] is `true` (the default), the test is put into [passed]
347 : /// if it succeeds. Otherwise, it's removed from [liveTests] entirely.
348 0 : Future _runLiveTest(LiveSuiteController suiteController, LiveTest liveTest,
349 : {bool countSuccess = true}) async {
350 0 : await _onUnpaused;
351 0 : _active.add(liveTest);
352 :
353 0 : var subscription = liveTest.onStateChange.listen(null);
354 : subscription
355 0 : ..onData((state) {
356 0 : if (state.status != Status.complete) return;
357 0 : _active.remove(liveTest);
358 : })
359 0 : ..onDone(() {
360 0 : _subscriptions.remove(subscription);
361 : });
362 0 : _subscriptions.add(subscription);
363 :
364 0 : suiteController.reportLiveTest(liveTest, countSuccess: countSuccess);
365 :
366 : // Schedule a microtask to ensure that [onTestStarted] fires before the
367 : // first [LiveTest.onStateChange] event.
368 0 : await Future.microtask(liveTest.run);
369 :
370 : // Once the test finishes, use [new Future] to do a coarse-grained event
371 : // loop pump to avoid starving non-microtask events.
372 0 : await Future(() {});
373 :
374 0 : if (!_restarted.contains(liveTest)) return;
375 0 : await _runLiveTest(suiteController, liveTest.copy(),
376 : countSuccess: countSuccess);
377 0 : _restarted.remove(liveTest);
378 : }
379 :
380 : /// Runs a dummy [LiveTest] for a test marked as "skip".
381 : ///
382 : /// [suiteController] is the controller for the suite that contains [test].
383 : /// [parents] is a list of groups that contain [test].
384 0 : Future _runSkippedTest(LiveSuiteController suiteController, Test test,
385 : List<Group> parents) async {
386 0 : await _onUnpaused;
387 0 : var skipped = LocalTest(test.name, test.metadata, () {}, trace: test.trace);
388 :
389 : late LiveTestController controller;
390 : controller =
391 0 : LiveTestController(suiteController.liveSuite.suite, skipped, () {
392 0 : controller.setState(const State(Status.running, Result.success));
393 0 : controller.setState(const State(Status.running, Result.skipped));
394 :
395 0 : if (skipped.metadata.skipReason != null) {
396 0 : controller
397 0 : .message(Message.skip('Skip: ${skipped.metadata.skipReason}'));
398 : }
399 :
400 0 : controller.setState(const State(Status.complete, Result.skipped));
401 0 : controller.completer.complete();
402 0 : }, () {}, groups: parents);
403 :
404 0 : return await _runLiveTest(suiteController, controller);
405 : }
406 :
407 : /// Closes [liveTest] and tells the engine to re-run it once it's done
408 : /// running.
409 : ///
410 : /// Returns the same future as [LiveTest.close].
411 0 : Future restartTest(LiveTest liveTest) async {
412 0 : if (_activeSuiteLoads.contains(liveTest)) {
413 0 : throw ArgumentError("Can't restart a load test.");
414 : }
415 :
416 0 : if (!_active.contains(liveTest)) {
417 0 : throw StateError("Can't restart inactive test "
418 0 : '"${liveTest.test.name}".');
419 : }
420 :
421 0 : _restarted.add(liveTest);
422 0 : _active.remove(liveTest);
423 0 : await liveTest.close();
424 : }
425 :
426 : /// Runs [suite] and returns the [LiveSuiteController] for the suite it loads.
427 : ///
428 : /// Returns `null` if the suite fails to load.
429 0 : Future<LiveSuiteController?> _addLoadSuite(LoadSuite suite) async {
430 0 : var controller = LiveSuiteController(suite);
431 0 : _addLiveSuite(controller.liveSuite);
432 :
433 0 : var liveTest = suite.test.load(suite);
434 0 : _activeSuiteLoads.add(liveTest);
435 :
436 0 : var subscription = liveTest.onStateChange.listen(null);
437 : subscription
438 0 : ..onData((state) {
439 0 : if (state.status != Status.complete) return;
440 0 : _activeSuiteLoads.remove(liveTest);
441 : })
442 0 : ..onDone(() {
443 0 : _subscriptions.remove(subscription);
444 : });
445 0 : _subscriptions.add(subscription);
446 :
447 0 : controller.reportLiveTest(liveTest, countSuccess: false);
448 0 : controller.noMoreLiveTests();
449 :
450 : // Schedule a microtask to ensure that [onTestStarted] fires before the
451 : // first [LiveTest.onStateChange] event.
452 0 : await Future.microtask(liveTest.run);
453 :
454 0 : var innerSuite = await suite.suite;
455 : if (innerSuite == null) return null;
456 :
457 0 : var innerController = LiveSuiteController(innerSuite);
458 0 : unawaited(innerController.liveSuite.onClose.whenComplete(() {
459 : // When the main suite is closed, close the load suite and its test as
460 : // well. This doesn't release any resources, but it does close streams
461 : // which indicates that the load test won't experience an error in the
462 : // future.
463 0 : liveTest.close();
464 0 : controller.close();
465 : }));
466 :
467 : return innerController;
468 : }
469 :
470 : /// Add [liveSuite] and the information it exposes to the engine's
471 : /// informational streams and collections.
472 0 : void _addLiveSuite(LiveSuite liveSuite) {
473 0 : _onSuiteStartedController.add(liveSuite);
474 :
475 0 : _onTestStartedGroup.add(liveSuite.onTestStarted);
476 0 : _passedGroup.add(liveSuite.passed);
477 0 : _skippedGroup.add(liveSuite.skipped);
478 0 : _failedGroup.add(liveSuite.failed);
479 : }
480 :
481 : /// Pauses the engine.
482 : ///
483 : /// This pauses all streams and keeps any new suites from being loaded or
484 : /// tests from being run until [resume] is called.
485 : ///
486 : /// This does nothing if the engine is already paused. Pauses are *not*
487 : /// cumulative.
488 0 : void pause() {
489 0 : if (_pauseCompleter != null) return;
490 0 : _pauseCompleter = Completer();
491 0 : for (var subscription in _subscriptions) {
492 0 : subscription.pause();
493 : }
494 : }
495 :
496 0 : void resume() {
497 0 : if (_pauseCompleter == null) return;
498 0 : _pauseCompleter!.complete();
499 0 : _pauseCompleter = null;
500 0 : for (var subscription in _subscriptions) {
501 0 : subscription.resume();
502 : }
503 : }
504 :
505 : /// Signals that the caller is done paying attention to test results and the
506 : /// engine should release any resources it has allocated.
507 : ///
508 : /// Any actively-running tests are also closed. VM tests are allowed to finish
509 : /// running so that any modifications they've made to the filesystem can be
510 : /// cleaned up.
511 : ///
512 : /// **Note that closing the engine is not the same as closing [suiteSink].**
513 : /// Closing [suiteSink] indicates that no more input will be provided, closing
514 : /// the engine indicates that no more output should be emitted.
515 0 : Future close() async {
516 0 : _closed = true;
517 0 : if (_closedBeforeDone != null) _closedBeforeDone = true;
518 0 : await _suiteController.close();
519 0 : await _onSuiteAddedController.close();
520 :
521 : // Close the running tests first so that we're sure to wait for them to
522 : // finish before we close their suites and cause them to become unloaded.
523 0 : var allLiveTests = liveTests.toSet()..addAll(_activeSuiteLoads);
524 0 : var futures = allLiveTests.map((liveTest) => liveTest.close()).toList();
525 :
526 : // Closing the run pool will close the test suites as soon as their tests
527 : // are done. For browser suites this is effectively immediate since their
528 : // tests shut down as soon as they're closed, but for VM suites we may need
529 : // to wait for tearDowns or tearDownAlls to run.
530 0 : futures.add(_runPool.close());
531 0 : await Future.wait(futures, eagerError: true);
532 : }
533 : }
|