From 8e425e2755a2ced73a35cf610715e235f92e7daa Mon Sep 17 00:00:00 2001 From: Jon Samwell Date: Thu, 1 Nov 2018 09:33:26 +1100 Subject: [PATCH] feat(reporter): add a test run summary reporter fix(example): fixed up glob pattern in example --- CHANGELOG.md | 5 + README.md | 19 ++-- example/test_driver/app_test.dart | 8 +- .../sub-features/counter_increases.feature | 8 ++ .../steps/tap_button_n_times_step.dart | 4 +- lib/flutter_gherkin.dart | 1 + .../flutter/flutter_run_process_handler.dart | 26 ++++-- lib/src/reporters/progress_reporter.dart | 12 +-- lib/src/reporters/stdout_reporter.dart | 15 ++- .../reporters/test_run_summary_reporter.dart | 91 +++++++++++++++++++ lib/src/test_runner.dart | 51 ++++++----- pubspec.yaml | 2 +- test/reporters/progress_reporter_test.dart | 58 ++++++++++++ .../test_run_summary_reporter_test.dart | 52 +++++++++++ 14 files changed, 298 insertions(+), 54 deletions(-) create mode 100644 example/test_driver/features/sub-features/counter_increases.feature create mode 100644 lib/src/reporters/test_run_summary_reporter.dart create mode 100644 test/reporters/progress_reporter_test.dart create mode 100644 test/reporters/test_run_summary_reporter_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c0c388..8a96057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.0.7] - 01/11/2018 + +* Added a test run summary reporter `TestRunSummaryReporter` that logs an aggregated summary of the test run once all tests have run. +* Fixed up glob issue in example project + ## [0.0.6] - 31/10/2018 * Added quick start steps in the example app readme diff --git a/README.md b/README.md index c6fb55d..61a84c9 100644 --- a/README.md +++ b/README.md @@ -148,8 +148,8 @@ import 'package:flutter_gherkin/flutter_gherkin.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/**/*.feature")] - ..reporters = [ProgressReporter()] + ..features = [Glob(r"test_driver/features/*.feature")] + ..reporters = [ProgressReporter(), TestRunSummaryReporter()] ..restartAppBetweenScenarios = true ..targetAppPath = "test_driver/app.dart" ..exitAfterTestRun = true; @@ -157,7 +157,7 @@ Future main() { } ``` -This code simple creates a configuration object and calls this library which will then promptly parse your feature files and run the tests. The configuration file is important and explained in further detail below. However, all that is happening is a `Glob` is provide which specifies the path to one or more feature files, it sets the reporter to the `ProgressReporter` report which mean prints to the result of a scenario and step to the standard output (console). Finally it specifies the path to the testable app created above `test_driver/app.dart`. This is important as it instructions the library which app to run the tests against. +This code simple creates a configuration object and calls this library which will then promptly parse your feature files and run the tests. The configuration file is important and explained in further detail below. However, all that is happening is a `Glob` is provide which specifies the path to one or more feature files, it sets the reporters to the `ProgressReporter` report which prints the result of scenarios and steps to the standard output (console). The `TestRunSummaryReporter` prints a summary of the run once all tests have been executed. Finally it specifies the path to the testable app created above `test_driver/app.dart`. This is important as it instructions the library which app to run the tests against. Finally to actually run the tests run the below on the command line: @@ -179,7 +179,7 @@ The parameters below can be specified in your configuration file: *Required* -An iterable of `Glob` patterns that specify the location(s) of `*.feature` files to run. +An iterable of `Glob` patterns that specify the location(s) of `*.feature` files to run. See #### tagExpression @@ -208,7 +208,7 @@ import 'steps/tap_button_n_times_step.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/**/*.feature")] + ..features = [Glob(r"test_driver/features/*.feature")] ..reporters = [StdoutReporter()] ..stepDefinitions = [TapButtonNTimesStep(), GivenIPickAColour()] ..restartAppBetweenScenarios = true @@ -234,7 +234,7 @@ import 'steps/tap_button_n_times_step.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/**/*.feature")] + ..features = [Glob(r"test_driver/features/*.feature")] ..reporters = [StdoutReporter()] ..stepDefinitions = [TapButtonNTimesStep(), GivenIPickAColour()] ..customStepParameterDefinitions = [ColourParameter()] @@ -272,7 +272,7 @@ import 'steps/tap_button_n_times_step.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/**/*.feature")] + ..features = [Glob(r"test_driver/features/*.feature")] ..reporters = [StdoutReporter()] ..stepDefinitions = [TapButtonNTimesStep(), GivenIPickAColour()] ..customStepParameterDefinitions = [ColourParameter()] @@ -299,7 +299,7 @@ import 'steps/tap_button_n_times_step.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/**/*.feature")] + ..features = [Glob(r"test_driver/features/*.feature")] ..reporters = [StdoutReporter()] ..stepDefinitions = [TapButtonNTimesStep(), GivenIPickAColour()] ..customStepParameterDefinitions = [ColourParameter()] @@ -426,7 +426,7 @@ class TapButtonNTimesStep extends When2WithWorld { @override Future executeStep(String input1, int input2) async { final locator = find.byValueKey(input1); - for (var i = 0; i < 10; i += 1) { + for (var i = 0; i < input2; i += 1) { await world.driver.tap(locator, timeout: timeout); } } @@ -673,6 +673,7 @@ A reporter is a class that is able to report on the progress of the test run. In - `StdoutReporter` - prints all messages from the test run to the console. - `ProgressReporter` - prints the result of each scenario and step to the console - colours the output. +- `TestRunSummaryReporter` - prints the results and duration of the test run once the run has completed - colours the output. You can create your own custom reporter by inheriting from the base `Reporter` class and overriding the one or many of the methods to direct the output message. The `Reporter` defines the following methods that can be overridden. All methods must return a `Future` and can be async. diff --git a/example/test_driver/app_test.dart b/example/test_driver/app_test.dart index 5b949e0..1d97548 100644 --- a/example/test_driver/app_test.dart +++ b/example/test_driver/app_test.dart @@ -8,10 +8,12 @@ import 'steps/tap_button_n_times_step.dart'; Future main() { final config = FlutterTestConfiguration() - ..features = [Glob(r"test_driver/features/*.feature")] + ..features = [Glob(r"test_driver/features/**.feature")] ..reporters = [ - ProgressReporter() - ] // you can include the "StdoutReporter()" for more verbose information + ProgressReporter(), + TestRunSummaryReporter(), + StdoutReporter(MessageLevel.warning) + ] // you can include the "StdoutReporter()" without the message level parameter for verbose log information ..hooks = [HookExample()] ..stepDefinitions = [TapButtonNTimesStep(), GivenIPickAColour()] ..customStepParameterDefinitions = [ColourParameter()] diff --git a/example/test_driver/features/sub-features/counter_increases.feature b/example/test_driver/features/sub-features/counter_increases.feature new file mode 100644 index 0000000..0bf7d1b --- /dev/null +++ b/example/test_driver/features/sub-features/counter_increases.feature @@ -0,0 +1,8 @@ +Feature: Counter + The counter should be incremented when the button is pressed. + + @perf + Scenario: Counter increases when the button is pressed + Given I expect the "counter" to be "0" + When I tap the "increment" button 20 times + Then I expect the "counter" to be "20" \ No newline at end of file diff --git a/example/test_driver/steps/tap_button_n_times_step.dart b/example/test_driver/steps/tap_button_n_times_step.dart index d1fc0f0..caa1134 100644 --- a/example/test_driver/steps/tap_button_n_times_step.dart +++ b/example/test_driver/steps/tap_button_n_times_step.dart @@ -8,8 +8,8 @@ class TapButtonNTimesStep extends When2WithWorld { @override Future executeStep(String input1, int input2) async { final locator = find.byValueKey(input1); - for (var i = 0; i < 10; i += 1) { - await world.driver.tap(locator, timeout: timeout); + for (var i = 0; i < input2; i += 1) { + await FlutterDriverUtils.tap(world.driver, locator, timeout: timeout); } } diff --git a/lib/flutter_gherkin.dart b/lib/flutter_gherkin.dart index 87e8ef5..9b00b16 100644 --- a/lib/flutter_gherkin.dart +++ b/lib/flutter_gherkin.dart @@ -22,6 +22,7 @@ export "src/reporters/message_level.dart"; export "src/reporters/messages.dart"; export "src/reporters/stdout_reporter.dart"; export "src/reporters/progress_reporter.dart"; +export "src/reporters/test_run_summary_reporter.dart"; // Hooks export "src/hooks/hook.dart"; diff --git a/lib/src/flutter/flutter_run_process_handler.dart b/lib/src/flutter/flutter_run_process_handler.dart index 8660661..ba8445e 100644 --- a/lib/src/flutter/flutter_run_process_handler.dart +++ b/lib/src/flutter/flutter_run_process_handler.dart @@ -11,6 +11,9 @@ class FlutterRunProcessHandler extends ProcessHandler { r"observatory debugger .*[:]? (http[s]?:.*\/).*", caseSensitive: false, multiLine: false); + + static RegExp _noConnectedDeviceRegex = + RegExp(r"no connected device", caseSensitive: false, multiLine: false); Process _runningProcess; Stream _processStdoutStream; List _openSubscriptions = List(); @@ -59,17 +62,24 @@ class FlutterRunProcessHandler extends ProcessHandler { _ensureRunningProcess(); final completer = Completer(); StreamSubscription sub; - sub = _processStdoutStream - .timeout(timeout, - onTimeout: (_) => completer.completeError(TimeoutException( - "Time out while wait for observatory debugger uri", timeout))) - .listen((logLine) { + sub = _processStdoutStream.timeout(timeout, onTimeout: (_) { + sub?.cancel(); + if (!completer.isCompleted) + completer.completeError(TimeoutException( + "Timeout while wait for observatory debugger uri", timeout)); + }).listen((logLine) { if (_observatoryDebuggerUriRegex.hasMatch(logLine)) { sub?.cancel(); - completer.complete( - _observatoryDebuggerUriRegex.firstMatch(logLine).group(1)); + if (!completer.isCompleted) + completer.complete( + _observatoryDebuggerUriRegex.firstMatch(logLine).group(1)); + } else if (_noConnectedDeviceRegex.hasMatch(logLine)) { + sub?.cancel(); + if (!completer.isCompleted) + stderr.writeln( + "${FAIL_COLOR}No connected devices found to run app on and tests against$RESET_COLOR"); } - }); + }, cancelOnError: true); return completer.future; } diff --git a/lib/src/reporters/progress_reporter.dart b/lib/src/reporters/progress_reporter.dart index 587e9d6..382c288 100644 --- a/lib/src/reporters/progress_reporter.dart +++ b/lib/src/reporters/progress_reporter.dart @@ -5,25 +5,23 @@ import 'package:flutter_gherkin/src/reporters/message_level.dart'; import 'package:flutter_gherkin/src/reporters/messages.dart'; class ProgressReporter extends StdoutReporter { - static const String PASS_COLOR = "\u001b[33;32m"; // green - @override Future onScenarioStarted(StartedMessage message) async { - printMessage( + printMessageLine( "Running scenario: ${_getNameAndContext(message.name, message.context)}", StdoutReporter.WARN_COLOR); } @override Future onScenarioFinished(ScenarioFinishedMessage message) async { - printMessage( + printMessageLine( "${message.passed ? 'PASSED' : 'FAILED'}: Scenario ${_getNameAndContext(message.name, message.context)}", - message.passed ? PASS_COLOR : StdoutReporter.FAIL_COLOR); + message.passed ? StdoutReporter.PASS_COLOR : StdoutReporter.FAIL_COLOR); } @override Future onStepFinished(StepFinishedMessage message) async { - printMessage( + printMessageLine( [ " ", _getStatePrefixIcon(message.result.result), @@ -72,7 +70,7 @@ class ProgressReporter extends StdoutReporter { String _getMessageColour(StepExecutionResult result) { switch (result) { case StepExecutionResult.pass: - return PASS_COLOR; + return StdoutReporter.PASS_COLOR; case StepExecutionResult.fail: return StdoutReporter.FAIL_COLOR; case StepExecutionResult.error: diff --git a/lib/src/reporters/stdout_reporter.dart b/lib/src/reporters/stdout_reporter.dart index 25d51ee..ffa7aa3 100644 --- a/lib/src/reporters/stdout_reporter.dart +++ b/lib/src/reporters/stdout_reporter.dart @@ -3,14 +3,20 @@ import 'package:flutter_gherkin/src/reporters/message_level.dart'; import 'package:flutter_gherkin/src/reporters/reporter.dart'; class StdoutReporter extends Reporter { + final MessageLevel _logLevel; static const String NEUTRAL_COLOR = "\u001b[33;34m"; // blue static const String DEBUG_COLOR = "\u001b[1;30m"; // gray static const String FAIL_COLOR = "\u001b[33;31m"; // red static const String WARN_COLOR = "\u001b[33;10m"; // yellow static const String RESET_COLOR = "\u001b[33;0m"; + static const String PASS_COLOR = "\u001b[33;32m"; // green + + StdoutReporter([this._logLevel = MessageLevel.verbose]); Future message(String message, MessageLevel level) async { - printMessage(message, getColour(level)); + if (level.index >= _logLevel.index) { + printMessageLine(message, getColour(level)); + } } String getColour(MessageLevel level) { @@ -28,8 +34,13 @@ class StdoutReporter extends Reporter { } } - void printMessage(String message, [String colour]) { + void printMessageLine(String message, [String colour]) { stdout.writeln( "${colour == null ? RESET_COLOR : colour}$message$RESET_COLOR"); } + + void printMessage(String message, [String colour]) { + stdout + .write("${colour == null ? RESET_COLOR : colour}$message$RESET_COLOR"); + } } diff --git a/lib/src/reporters/test_run_summary_reporter.dart b/lib/src/reporters/test_run_summary_reporter.dart new file mode 100644 index 0000000..57a92d4 --- /dev/null +++ b/lib/src/reporters/test_run_summary_reporter.dart @@ -0,0 +1,91 @@ +import 'package:flutter_gherkin/flutter_gherkin.dart'; +import 'package:flutter_gherkin/src/gherkin/steps/step_run_result.dart'; +import 'package:flutter_gherkin/src/reporters/message_level.dart'; +import 'package:flutter_gherkin/src/reporters/messages.dart'; + +class TestRunSummaryReporter extends StdoutReporter { + final _timer = new Stopwatch(); + final List _ranSteps = List(); + final List _ranScenarios = + List(); + + @override + Future onScenarioFinished(ScenarioFinishedMessage message) async { + _ranScenarios.add(message); + } + + @override + Future onStepFinished(StepFinishedMessage message) async { + _ranSteps.add(message); + } + + @override + Future message(String message, MessageLevel level) async { + // ignore messages + } + + @override + Future onTestRunStarted() async { + _timer.start(); + } + + @override + Future onTestRunFinished() async { + _timer.stop(); + printMessageLine( + "${_ranScenarios.length} scenario${_ranScenarios.length > 1 ? "s" : ""} (${_collectScenarioSummary(_ranScenarios)})"); + printMessageLine( + "${_ranSteps.length} step${_ranSteps.length > 1 ? "s" : ""} (${_collectStepSummary(_ranSteps)})"); + printMessageLine("${Duration(milliseconds: _timer.elapsedMilliseconds)}"); + } + + @override + Future dispose() async { + if (_timer.isRunning) { + _timer.stop(); + } + } + + String _collectScenarioSummary(Iterable scenarios) { + List summaries = List(); + if (scenarios.any((s) => s.passed)) { + summaries.add( + "${StdoutReporter.PASS_COLOR}${scenarios.where((s) => s.passed).length} passed${StdoutReporter.RESET_COLOR}"); + } + + if (scenarios.any((s) => !s.passed)) { + summaries.add( + "${StdoutReporter.FAIL_COLOR}${scenarios.where((s) => !s.passed).length} failed${StdoutReporter.RESET_COLOR}"); + } + + return summaries.join(", "); + } + + String _collectStepSummary(Iterable steps) { + List summaries = List(); + final passed = + steps.where((s) => s.result.result == StepExecutionResult.pass); + final skipped = + steps.where((s) => s.result.result == StepExecutionResult.skipped); + final failed = steps.where((s) => + s.result.result == StepExecutionResult.error || + s.result.result == StepExecutionResult.fail || + s.result.result == StepExecutionResult.timeout); + if (passed.length > 0) { + summaries.add( + "${StdoutReporter.PASS_COLOR}${passed.length} passed${StdoutReporter.RESET_COLOR}"); + } + + if (skipped.length > 0) { + summaries.add( + "${StdoutReporter.WARN_COLOR}${skipped.length} skipped${StdoutReporter.RESET_COLOR}"); + } + + if (failed.length > 0) { + summaries.add( + "${StdoutReporter.FAIL_COLOR}${failed.length} failed${StdoutReporter.RESET_COLOR}"); + } + + return summaries.join(", "); + } +} diff --git a/lib/src/test_runner.dart b/lib/src/test_runner.dart index af2415f..8824654 100644 --- a/lib/src/test_runner.dart +++ b/lib/src/test_runner.dart @@ -46,33 +46,40 @@ class GherkinRunner { } } - await _reporter.message( - "Found ${featureFiles.length} feature file(s) to run", - MessageLevel.info); - - if (config.order == ExecutionOrder.random) { - await _reporter.message( - "Executing features in random order", MessageLevel.info); - featureFiles = featureFiles.toList()..shuffle(); - } - - await _hook.onBeforeRun(config); - bool allFeaturesPassed = true; - try { - await _reporter.onTestRunStarted(); - for (var featureFile in featureFiles) { - final runner = new FeatureFileRunner(config, _tagExpressionEvaluator, - _executableSteps, _reporter, _hook); - await runner.run(featureFile); + + if (featureFiles.length == 0) { + await _reporter.message( + "No feature files found to run, exitting without running any scenarios", + MessageLevel.warning); + } else { + await _reporter.message( + "Found ${featureFiles.length} feature file(s) to run", + MessageLevel.info); + + if (config.order == ExecutionOrder.random) { + await _reporter.message( + "Executing features in random order", MessageLevel.info); + featureFiles = featureFiles.toList()..shuffle(); } - } finally { - await _reporter.onTestRunFinished(); + + await _hook.onBeforeRun(config); + + try { + await _reporter.onTestRunStarted(); + for (var featureFile in featureFiles) { + final runner = new FeatureFileRunner(config, _tagExpressionEvaluator, + _executableSteps, _reporter, _hook); + await runner.run(featureFile); + } + } finally { + await _reporter.onTestRunFinished(); + } + + await _hook.onAfterRun(config); } - await _hook.onAfterRun(config); await _reporter.dispose(); - exitCode = allFeaturesPassed ? 0 : 1; if (config.exitAfterTestRun) exit(allFeaturesPassed ? 0 : 1); diff --git a/pubspec.yaml b/pubspec.yaml index b5126e8..ba5061a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: flutter_gherkin description: A Gherkin / Cucumber parser and test runner for Dart and Flutter -version: 0.0.6 +version: 0.0.7 author: Jon Samwell homepage: https://github.com/jonsamwell/flutter_gherkin diff --git a/test/reporters/progress_reporter_test.dart b/test/reporters/progress_reporter_test.dart new file mode 100644 index 0000000..14ae8a9 --- /dev/null +++ b/test/reporters/progress_reporter_test.dart @@ -0,0 +1,58 @@ +import "package:flutter_gherkin/flutter_gherkin.dart"; +import "package:flutter_gherkin/src/gherkin/runnables/debug_information.dart"; +import "package:flutter_gherkin/src/gherkin/steps/step_run_result.dart"; +import "package:test/test.dart"; + +class TestableProgressReporter extends ProgressReporter { + final output = List(); + @override + void printMessageLine(String message, [String colour]) { + output.add(message); + } +} + +void main() { + group("report", () { + test("provides correct step finished output", () async { + final reporter = TestableProgressReporter(); + + await reporter.onStepFinished(StepFinishedMessage( + "Step 1", + RunnableDebugInformation("filePath", 1, "line 1"), + StepResult(0, StepExecutionResult.pass))); + await reporter.onStepFinished(StepFinishedMessage( + "Step 2", + RunnableDebugInformation("filePath", 2, "line 2"), + StepResult(0, StepExecutionResult.fail))); + await reporter.onStepFinished(StepFinishedMessage( + "Step 3", + RunnableDebugInformation("filePath", 3, "line 3"), + StepResult(0, StepExecutionResult.skipped))); + await reporter.onStepFinished(StepFinishedMessage( + "Step 4", + RunnableDebugInformation("filePath", 4, "line 4"), + StepResult(0, StepExecutionResult.error))); + await reporter.onStepFinished(StepFinishedMessage( + "Step 5", + RunnableDebugInformation("filePath", 5, "line 5"), + StepResult(1, StepExecutionResult.timeout))); + + expect(reporter.output, [ + " √ Step 1 # filePath:1 took 0ms ", + " × Step 2 # filePath:2 took 0ms ", + " - Step 3 # filePath:3 took 0ms ", + " × Step 4 # filePath:4 took 0ms ", + " × Step 5 # filePath:5 took 1ms " + ]); + }); + + test("provides correct scenerio started output", () async { + final reporter = TestableProgressReporter(); + + await reporter.onScenarioStarted(StartedMessage(Target.scenario, + "Scenerio 1", RunnableDebugInformation("filePath", 1, "line 1"))); + + expect(reporter.output, ["Running scenario: Scenerio 1 # filePath:1"]); + }); + }); +} diff --git a/test/reporters/test_run_summary_reporter_test.dart b/test/reporters/test_run_summary_reporter_test.dart new file mode 100644 index 0000000..3b5d827 --- /dev/null +++ b/test/reporters/test_run_summary_reporter_test.dart @@ -0,0 +1,52 @@ +import 'package:flutter_gherkin/flutter_gherkin.dart'; +import 'package:flutter_gherkin/src/gherkin/steps/step_run_result.dart'; +import 'package:test/test.dart'; + +class TestableTestRunSummaryReporter extends TestRunSummaryReporter { + final output = List(); + @override + void printMessageLine(String message, [String colour]) { + output.add(message); + } +} + +void main() { + group("report", () { + test("provides correct output", () async { + final reporter = TestableTestRunSummaryReporter(); + + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.pass))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.fail))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.skipped))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.skipped))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.pass))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.error))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.pass))); + await reporter.onStepFinished(StepFinishedMessage( + "", null, StepResult(0, StepExecutionResult.timeout))); + + await reporter + .onScenarioFinished(ScenarioFinishedMessage("", null, true)); + await reporter + .onScenarioFinished(ScenarioFinishedMessage("", null, false)); + await reporter + .onScenarioFinished(ScenarioFinishedMessage("", null, false)); + await reporter + .onScenarioFinished(ScenarioFinishedMessage("", null, true)); + + await reporter.onTestRunFinished(); + expect(reporter.output, [ + "4 scenarios (\x1B[33;32m2 passed\x1B[33;0m, \x1B[33;31m2 failed\x1B[33;0m)", + "8 steps (\x1B[33;32m3 passed\x1B[33;0m, \x1B[33;10m2 skipped\x1B[33;0m, \x1B[33;31m3 failed\x1B[33;0m)", + "0:00:00.000000" + ]); + }); + }); +}