Line data Source code
1 : // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
2 : // for details. All rights reserved. Use of this source code is governed by a
3 : // BSD-style license that can be found in the LICENSE file.
4 :
5 : import 'dart:async';
6 :
7 : import 'package:stack_trace/stack_trace.dart';
8 :
9 : import '../frontend/expect.dart';
10 : import '../runner/load_suite.dart';
11 : import '../utils.dart';
12 : import 'closed_exception.dart';
13 : import 'group.dart';
14 : import 'live_test.dart';
15 : import 'live_test_controller.dart';
16 : import 'message.dart';
17 : import 'metadata.dart';
18 : import 'operating_system.dart';
19 : import 'outstanding_callback_counter.dart';
20 : import 'state.dart';
21 : import 'suite.dart';
22 : import 'test.dart';
23 : import 'test_platform.dart';
24 :
25 : /// A test in this isolate.
26 : class LocalTest extends Test {
27 : final String name;
28 : final Metadata metadata;
29 : final Trace trace;
30 :
31 : /// The test body.
32 : final AsyncFunction _body;
33 :
34 5 : LocalTest(this.name, this.metadata, body(), {this.trace}) : _body = body;
35 :
36 : /// Loads a single runnable instance of this test.
37 : LiveTest load(Suite suite, {Iterable<Group> groups}) {
38 5 : var invoker = new Invoker._(suite, this, groups: groups);
39 5 : return invoker.liveTest;
40 : }
41 :
42 : Test forPlatform(TestPlatform platform, {OperatingSystem os}) {
43 15 : if (!metadata.testOn.evaluate(platform, os: os)) return null;
44 25 : return new LocalTest(name, metadata.forPlatform(platform, os: os), _body,
45 5 : trace: trace);
46 : }
47 : }
48 :
49 : /// The class responsible for managing the lifecycle of a single local test.
50 : ///
51 : /// The current invoker is accessible within the zone scope of the running test
52 : /// using [Invoker.current]. It's used to track asynchronous callbacks and
53 : /// report asynchronous errors.
54 : class Invoker {
55 : /// The live test being driven by the invoker.
56 : ///
57 : /// This provides a view into the state of the test being executed.
58 10 : LiveTest get liveTest => _controller.liveTest;
59 : LiveTestController _controller;
60 :
61 : /// Whether the test can be closed in the current zone.
62 15 : bool get _closable => Zone.current[_closableKey];
63 :
64 : /// An opaque object used as a key in the zone value map to identify
65 : /// [_closable].
66 : ///
67 : /// This is an instance variable to ensure that multiple invokers don't step
68 : /// on one anothers' toes.
69 : final _closableKey = new Object();
70 :
71 : /// Whether the test has been closed.
72 : ///
73 : /// Once the test is closed, [expect] and [expectAsync] will throw
74 : /// [ClosedException]s whenever accessed to help the test stop executing as
75 : /// soon as possible.
76 15 : bool get closed => _closable && _onCloseCompleter.isCompleted;
77 :
78 : /// A future that completes once the test has been closed.
79 0 : Future get onClose => _closable
80 0 : ? _onCloseCompleter.future
81 : // If we're in an unclosable block, return a future that will never
82 : // complete.
83 0 : : new Completer().future;
84 : final _onCloseCompleter = new Completer();
85 :
86 : /// The test being run.
87 15 : LocalTest get _test => liveTest.test as LocalTest;
88 :
89 : /// The outstanding callback counter for the current zone.
90 : OutstandingCallbackCounter get _outstandingCallbacks {
91 15 : var counter = Zone.current[_counterKey];
92 : if (counter != null) return counter;
93 0 : throw new StateError("Can't add or remove outstanding callbacks outside "
94 : "of a test body.");
95 : }
96 :
97 : /// All the zones created by [waitForOutstandingCallbacks], in the order they
98 : /// were created.
99 : ///
100 : /// This is used to throw timeout errors in the most recent zone.
101 : final _outstandingCallbackZones = <Zone>[];
102 :
103 : /// An opaque object used as a key in the zone value map to identify
104 : /// [_outstandingCallbacks].
105 : ///
106 : /// This is an instance variable to ensure that multiple invokers don't step
107 : /// on one anothers' toes.
108 : final _counterKey = new Object();
109 :
110 : /// The number of times this [liveTest] has been run.
111 : int _runCount = 0;
112 :
113 : /// The current invoker, or `null` if none is defined.
114 : ///
115 : /// An invoker is only set within the zone scope of a running test.
116 : static Invoker get current {
117 : // TODO(nweiz): Use a private symbol when dart2js supports it (issue 17526).
118 10 : return Zone.current[#test.invoker];
119 : }
120 :
121 : /// The zone that the top level of [_test.body] is running in.
122 : ///
123 : /// Tracking this ensures that [_timeoutTimer] isn't created in a
124 : /// timer-mocking zone created by the test.
125 : Zone _invokerZone;
126 :
127 : /// The timer for tracking timeouts.
128 : ///
129 : /// This will be `null` until the test starts running.
130 : Timer _timeoutTimer;
131 :
132 : /// The tear-down functions to run when this test finishes.
133 : final _tearDowns = <AsyncFunction>[];
134 :
135 : /// Messages to print if and when this test fails.
136 : final _printsOnFailure = <String>[];
137 :
138 5 : Invoker._(Suite suite, LocalTest test, {Iterable<Group> groups}) {
139 10 : _controller = new LiveTestController(
140 15 : suite, test, _onRun, _onCloseCompleter.complete,
141 : groups: groups);
142 : }
143 :
144 : /// Runs [callback] after this test completes.
145 : ///
146 : /// The [callback] may return a [Future]. Like all tear-downs, callbacks are
147 : /// run in the reverse of the order they're declared.
148 : void addTearDown(callback()) {
149 0 : if (closed) throw new ClosedException();
150 0 : _tearDowns.add(callback);
151 : }
152 :
153 : /// Tells the invoker that there's a callback running that it should wait for
154 : /// before considering the test successful.
155 : ///
156 : /// Each call to [addOutstandingCallback] should be followed by a call to
157 : /// [removeOutstandingCallback] once the callbak is no longer running. Note
158 : /// that only successful tests wait for outstanding callbacks; as soon as a
159 : /// test experiences an error, any further calls to [addOutstandingCallback]
160 : /// or [removeOutstandingCallback] will do nothing.
161 : ///
162 : /// Throws a [ClosedException] if this test has been closed.
163 : void addOutstandingCallback() {
164 0 : if (closed) throw new ClosedException();
165 0 : _outstandingCallbacks.addOutstandingCallback();
166 : }
167 :
168 : /// Tells the invoker that a callback declared with [addOutstandingCallback]
169 : /// is no longer running.
170 : void removeOutstandingCallback() {
171 5 : heartbeat();
172 10 : _outstandingCallbacks.removeOutstandingCallback();
173 : }
174 :
175 : /// Removes all outstanding callbacks, for example when an error occurs.
176 : ///
177 : /// Future calls to [addOutstandingCallback] and [removeOutstandingCallback]
178 : /// will be ignored.
179 : void removeAllOutstandingCallbacks() =>
180 0 : _outstandingCallbacks.removeAllOutstandingCallbacks();
181 :
182 : /// Runs [fn] and returns once all (registered) outstanding callbacks it
183 : /// transitively invokes have completed.
184 : ///
185 : /// If [fn] itself returns a future, this will automatically wait until that
186 : /// future completes as well. Note that outstanding callbacks registered
187 : /// within [fn] will *not* be registered as outstanding callback outside of
188 : /// [fn].
189 : ///
190 : /// If [fn] produces an unhandled error, this marks the current test as
191 : /// failed, removes all outstanding callbacks registered within [fn], and
192 : /// completes the returned future. It does not remove any outstanding
193 : /// callbacks registered outside of [fn].
194 : ///
195 : /// If the test times out, the *most recent* call to
196 : /// [waitForOutstandingCallbacks] will treat that error as occurring within
197 : /// [fn]—that is, it will complete immediately.
198 : Future waitForOutstandingCallbacks(fn()) {
199 5 : heartbeat();
200 :
201 : var zone;
202 5 : var counter = new OutstandingCallbackCounter();
203 5 : runZoned(() async {
204 5 : zone = Zone.current;
205 10 : _outstandingCallbackZones.add(zone);
206 10 : await fn();
207 5 : counter.removeOutstandingCallback();
208 10 : }, zoneValues: {_counterKey: counter});
209 :
210 10 : return counter.noOutstandingCallbacks.whenComplete(() {
211 10 : _outstandingCallbackZones.remove(zone);
212 : });
213 : }
214 :
215 : /// Runs [fn] in a zone where [closed] is always `false`.
216 : ///
217 : /// This is useful for running code that should be able to register callbacks
218 : /// and interact with the test framework normally even when the invoker is
219 : /// closed, for example cleanup code.
220 : unclosable(fn()) {
221 5 : heartbeat();
222 :
223 15 : return runZoned(fn, zoneValues: {_closableKey: false});
224 : }
225 :
226 : /// Notifies the invoker that progress is being made.
227 : ///
228 : /// Each heartbeat resets the timeout timer. This helps ensure that
229 : /// long-running tests that still make progress don't time out.
230 : void heartbeat() {
231 10 : if (liveTest.isComplete) return;
232 15 : if (_timeoutTimer != null) _timeoutTimer.cancel();
233 :
234 : var timeout =
235 30 : liveTest.test.metadata.timeout.apply(new Duration(seconds: 30));
236 : if (timeout == null) return;
237 15 : _timeoutTimer = _invokerZone.createTimer(timeout, () {
238 0 : _outstandingCallbackZones.last.run(() {
239 0 : if (liveTest.isComplete) return;
240 0 : _handleError(
241 0 : Zone.current,
242 0 : new TimeoutException(
243 0 : "Test timed out after ${niceDuration(timeout)}.", timeout));
244 : });
245 : });
246 : }
247 :
248 : /// Marks the current test as skipped.
249 : ///
250 : /// If passed, [message] is emitted as a skip message.
251 : ///
252 : /// Note that this *does not* mark the test as complete. That is, it sets
253 : /// the result to [Result.skipped], but doesn't change the state.
254 : void skip([String message]) {
255 0 : if (liveTest.state.shouldBeDone) {
256 : // Set the state explicitly so we don't get an extra error about the test
257 : // failing after being complete.
258 0 : _controller.setState(const State(Status.complete, Result.error));
259 : throw "This test was marked as skipped after it had already completed. "
260 : "Make sure to use\n"
261 : "[expectAsync] or the [completes] matcher when testing async code.";
262 : }
263 :
264 0 : if (message != null) _controller.message(new Message.skip(message));
265 : // TODO: error if the test is already complete.
266 0 : _controller.setState(const State(Status.pending, Result.skipped));
267 : }
268 :
269 : /// Prints [message] if and when this test fails.
270 : void printOnFailure(String message) {
271 0 : message = message.trim();
272 0 : if (liveTest.state.result.isFailing) {
273 0 : print("\n$message");
274 : } else {
275 0 : _printsOnFailure.add(message);
276 : }
277 : }
278 :
279 : /// Notifies the invoker of an asynchronous error.
280 : ///
281 : /// The [zone] is the zone in which the error was thrown.
282 : void _handleError(Zone zone, error, [StackTrace stackTrace]) {
283 : // Ignore errors propagated from previous test runs
284 0 : if (_runCount != zone[#runCount]) return;
285 0 : if (stackTrace == null) stackTrace = new Chain.current();
286 :
287 : // Store these here because they'll change when we set the state below.
288 0 : var shouldBeDone = liveTest.state.shouldBeDone;
289 :
290 0 : if (error is! TestFailure) {
291 0 : _controller.setState(const State(Status.complete, Result.error));
292 0 : } else if (liveTest.state.result != Result.error) {
293 0 : _controller.setState(const State(Status.complete, Result.failure));
294 : }
295 :
296 0 : _controller.addError(error, stackTrace);
297 0 : zone.run(removeAllOutstandingCallbacks);
298 :
299 0 : if (!liveTest.test.metadata.chainStackTraces) {
300 0 : _printsOnFailure.add("Consider enabling the flag chain-stack-traces to "
301 : "receive more detailed exceptions.\n"
302 : "For example, 'pub run test --chain-stack-traces'.");
303 : }
304 :
305 0 : if (_printsOnFailure.isNotEmpty) {
306 0 : print(_printsOnFailure.join("\n\n"));
307 0 : _printsOnFailure.clear();
308 : }
309 :
310 : // If a test was supposed to be done but then had an error, that indicates
311 : // that it was poorly-written and could be flaky.
312 : if (!shouldBeDone) return;
313 :
314 : // However, users don't think of load tests as "tests", so the error isn't
315 : // helpful for them.
316 : //
317 : // TODO(nweiz): Find a way of avoiding this error that doesn't require
318 : // Invoker to refer to a class from the runner.
319 0 : if (liveTest.suite is LoadSuite) return;
320 :
321 0 : _handleError(
322 : zone,
323 : "This test failed after it had already completed. Make sure to use "
324 : "[expectAsync]\n"
325 : "or the [completes] matcher when testing async code.",
326 : stackTrace);
327 : }
328 :
329 : /// The method that's run when the test is started.
330 : void _onRun() {
331 10 : _controller.setState(const State(Status.running, Result.success));
332 :
333 5 : var outstandingCallbacksForBody = new OutstandingCallbackCounter();
334 :
335 10 : _runCount++;
336 5 : Chain.capture(() {
337 5 : runZoned(() async {
338 10 : _invokerZone = Zone.current;
339 15 : _outstandingCallbackZones.add(Zone.current);
340 :
341 : // Run the test asynchronously so that the "running" state change has
342 : // a chance to hit its event handler(s) before the test produces an
343 : // error. If an error is emitted before the first state change is
344 : // handled, we can end up with [onError] callbacks firing before the
345 : // corresponding [onStateChkange], which violates the timing
346 : // guarantees.
347 : //
348 : // Using [new Future] also avoids starving the DOM or other
349 : // microtask-level events.
350 5 : new Future(() async {
351 15 : await _test._body();
352 15 : await unclosable(_runTearDowns);
353 5 : removeOutstandingCallback();
354 0 : });
355 :
356 15 : await _outstandingCallbacks.noOutstandingCallbacks;
357 15 : if (_timeoutTimer != null) _timeoutTimer.cancel();
358 :
359 20 : if (liveTest.state.result != Result.success &&
360 0 : _runCount < liveTest.test.metadata.retry + 1) {
361 0 : _controller
362 0 : .message(new Message.print("Retry: ${liveTest.test.name}"));
363 0 : _onRun();
364 : return;
365 : }
366 :
367 30 : _controller.setState(new State(Status.complete, liveTest.state.result));
368 :
369 15 : _controller.completer.complete();
370 0 : },
371 5 : zoneValues: {
372 : #test.invoker: this,
373 : // Use the invoker as a key so that multiple invokers can have different
374 : // outstanding callback counters at once.
375 5 : _counterKey: outstandingCallbacksForBody,
376 5 : _closableKey: true,
377 5 : #runCount: _runCount
378 : },
379 5 : zoneSpecification: new ZoneSpecification(
380 : print: (self, parent, zone, line) =>
381 3 : _controller.message(new Message.print(line)),
382 : // Use [handleUncaughtError] rather than [onError] so we can
383 : // capture [zone] and with it the outstanding callback counter for
384 : // the zone in which [error] was thrown.
385 : handleUncaughtError: (self, _, zone, error, stackTrace) => self
386 0 : .parent
387 0 : .run(() => _handleError(zone, error, stackTrace))));
388 20 : }, when: liveTest.test.metadata.chainStackTraces);
389 : }
390 :
391 : /// Run [_tearDowns] in reverse order.
392 : Future _runTearDowns() async {
393 10 : while (_tearDowns.isNotEmpty) {
394 0 : await errorsDontStopTest(_tearDowns.removeLast());
395 : }
396 0 : }
397 : }
|