feat(runner): app is not hot reloaded to increase testing speed

feat(build): no-build and flavor options now work
chore(lib): upgraded to v1 of dart_gherkin

CHANGELOG.md

@@ -1,3 +1,9 @@
+## [1.0.0] - 05/06/2019
+* Huge speed improvement when running tests by hot reloading (which clears the state) rather than restarting the app
+* Added flag to determine if the application should be built prior to running tests
+* Merged PR which allows for build flavor and device id to be specified thanks to @iqbalmineraltown for the PR
+* Updated to latest v1 dart_gherkin lib
+
 ## [0.0.14] - 23/04/2019
 * Updated to rely on the abstracted Gherkin library 'https://github.com/jonsamwell/dart_gherkin' which now includes a JsonReporter
 * Updated docs

README.md

@@ -38,6 +38,9 @@ Available as a Dart package https://pub.dartlang.org/packages/flutter_gherkin
     - [exitAfterTestRun](#exitaftertestrun)
   - [Flutter specific configuration options](#flutter-specific-configuration-options)
     - [restartAppBetweenScenarios](#restartappbetweenscenarios)
+    - [build](#build)
+    - [buildFlavor](#buildFlavor)
+    - [targetDeviceId](#targetDeviceId)
     - [targetAppPath](#targetapppath)
 - [Features Files](#features-files)
   - [Steps Definitions](#steps-definitions)
@@ -343,13 +346,19 @@ The `FlutterTestConfiguration` will automatically create some default Flutter op
 
 Defaults to `true`.
 
-To avoid tests starting on an app changed by a previous test it is suggested that the Flutter application under test be restarted between each scenario.  While this will increase the execution time slightly it will limit tests failing because they run against an app changed by a previous test.  Note in more complex application it may also be necessary to use the `AfterScenario` hook to reset the application to a base state a test can run on.  Logging out for example if restarting an application will present a lock screen etc.
+To avoid tests starting on an app changed by a previous test it is suggested that the Flutter application under test be restarted between each scenario.  While this will increase the execution time slightly it will limit tests failing because they run against an app changed by a previous test.  Note in more complex application it may also be necessary to use the `AfterScenario` hook to reset the application to a base state a test can run on.  Logging out for example if restarting an application will present a lock screen etc.  This now performs a hot reload of the application which resets the state and drastically reduces the time to run the tests.
 
 #### targetAppPath
 
 Defaults to `lib/test_driver/app.dart`
 
-This should point to the *testable* application that enables the Flutter driver extensions and thus is able to be automated.  This application wil be started when the test run in started and restarted if the `restartAppBetweenScenarios` configuration property is set to true.
+This should point to the *testable* application that enables the Flutter driver extensions and thus is able to be automated.  This application wil be started when the test run is started and restarted if the `restartAppBetweenScenarios` configuration property is set to true.
+
+#### build
+
+Defaults to `true`
+
+This optional argument lets you specify if the target application should be built prior to running the first test.  This defaults to `true`
 
 #### buildFlavor
 

analysis_options.yaml

@@ -3,82 +3,12 @@
 # see https://github.com/dart-lang/pedantic#enabled-lints.
 include: package:pedantic/analysis_options.yaml
 
-analyzer:
-  errors:
-    # treat missing required parameters as a warning (not a hint)
-    missing_required_param: warning
-    # treat missing returns as a warning (not a hint)
-    missing_return: warning
-    # allow having TODOs in the code
-    todo: ignore
-  exclude:
-    # - path/to/excluded/files/**
-
 # For lint rules and documentation, see http://dart-lang.github.io/linter/lints.
-linter:
+# Uncomment to specify additional rules.
-  rules:
+# linter:
-    # these rules are documented on and in the same order as
+#   rules:
-    # the Dart Lint rules page to make maintenance easier
+#     - camel_case_types
-    # http://dart-lang.github.io/linter/lints/
 
-    # === error rules ===
+# analyzer:
-    - avoid_empty_else
+#   exclude:
-    - avoid_slow_async_io
+#     - path/to/excluded/files/**
-    - cancel_subscriptions
-    - close_sinks
-    - control_flow_in_finally
-    - empty_statements
-    - hash_and_equals
-    - iterable_contains_unrelated_type
-    - list_remove_unrelated_type
-    - no_adjacent_strings_in_list
-    - no_duplicate_case_values
-    - test_types_in_equals
-    - throw_in_finally
-    - unrelated_type_equality_checks
-    - valid_regexps
-
-    # === style rules ===
-    - always_declare_return_types
-    - always_require_non_null_named_parameters
-    # - always_specify_types
-    - annotate_overrides
-    # - avoid_as
-    - avoid_init_to_null
-    - avoid_null_checks_in_equality_operators
-    - avoid_return_types_on_setters
-    - await_only_futures
-    - camel_case_types
-    # - directives_ordering
-    # - empty_catches
-    - empty_constructor_bodies
-    - implementation_imports
-    - library_names
-    - library_prefixes
-    - non_constant_identifier_names
-    - overridden_fields
-    - package_api_docs
-    - package_prefixed_library_names
-    - prefer_adjacent_string_concatenation
-    - prefer_collection_literals
-    - prefer_const_constructors
-    - prefer_contains
-    - prefer_equal_for_default_values
-    - prefer_final_locals
-    - prefer_initializing_formals
-    - prefer_is_empty
-    - prefer_is_not_empty
-    - prefer_void_to_null
-    - slash_for_doc_comments
-    # - sort_constructors_first
-    - sort_unnamed_constructors_first
-    - type_init_formals
-    - unnecessary_brace_in_string_interps
-    - unnecessary_const
-    - unnecessary_getters_setters
-    - unnecessary_new
-    - unnecessary_null_aware_assignments
-    - unnecessary_null_in_if_null_operators
-    - unnecessary_statements
-    - unnecessary_this
-    - use_rethrow_when_possible

example/README.md

@@ -17,5 +17,6 @@ This will run the features files found in the folder `test_driver/features` agai
 To debug this example and step through the library code.
 
 1. Set a break point in `test_driver/app_test.dart`
-2. If you are in VsCode you will simply be able to select `Debug example` from the dropdown in the `debugging tab` as the `launch.json` has been configured.
+2. Set `exitAfterTestRun` on the configuration to false to ensure exiting cleanly as the IDE will handle exiting
+3. If you are in VsCode you will simply be able to select `Debug example` from the dropdown in the `debugging tab` as the `launch.json` has been configured.
     - otherwise you will need to run a debugging session against `test_driver/app_test.dart`.

example/android/app/build.gradle

@@ -49,13 +49,16 @@ android {
         }
     }
 
-    /// uncomment to see example of using flavor
+    /// uncomment to see example of using flavors
-
     // flavorDimensions "env"
     // productFlavors {
     //     staging {
     //         dimension "env"
     //         applicationIdSuffix ".dev"
+    //         versionNameSuffix "-dev"
+    //     }
+    //     production {
+    //         dimension "env"
     //     }
     // }
 }

example/pubspec.yaml

@@ -17,7 +17,7 @@ dependencies:
     sdk: flutter
   path: ^1.6.2
   glob: ^1.1.7
-  gherkin: ^0.0.4
+  gherkin: ^1.0.0
 
 
   # The following adds the Cupertino Icons font to your application.
@@ -32,7 +32,6 @@ dev_dependencies:
   flutter_gherkin:
     path: ../
 
-
 # For information on the generic Dart part of this file, see the
 # following page: https://www.dartlang.org/tools/pub/pubspec
 

example/test_driver/app.dart

@@ -1,5 +1,4 @@
-import 'package:example/main.dart';
+import 'package:example/main.dart' as app;
-import 'package:flutter/widgets.dart';
 import 'package:flutter_driver/driver_extension.dart';
 
 void main() {
@@ -8,5 +7,5 @@ void main() {
 
   // Call the `main()` function of your app or call `runApp` with any widget you
   // are interested in testing.
-  runApp(MyApp());
+  app.main();
 }

example/test_driver/app_test.dart

@@ -20,7 +20,7 @@ Future<void> main() {
     ..customStepParameterDefinitions = [ColourParameter()]
     ..restartAppBetweenScenarios = true
     ..targetAppPath = "test_driver/app.dart"
-    // ..buildFlavor = "staging" // uncomment when using build flavor and check android/ios flavor setup
+    // ..buildFlavor = "staging" // uncomment when using build flavor and check android/ios flavor setup see android file android\app\build.gradle
     // ..targetDeviceId = "all" // uncomment to run tests on all connected devices or set specific device target id
     // ..tagExpression = "@smoke" // uncomment to see an example of running scenarios based on tag expressions
     ..exitAfterTestRun = true; // set to false if debugging to exit cleanly

example/test_driver/steps/data_table_example_step.dart

@@ -12,7 +12,7 @@ import 'package:gherkin/gherkin.dart';
 class GivenIAddTheUsers extends Given1<Table> {
   @override
   Future<void> executeStep(Table dataTable) async {
-    // TODO: implement executeStep
+    // implement executeStep
     for (var row in dataTable.rows) {
       // do something with row
       row.columns.forEach((columnValue) => print(columnValue));

example/test_driver/steps/multiline_string_example_step.dart

@@ -11,7 +11,7 @@ import 'package:gherkin/gherkin.dart';
 class GivenIProvideAComment extends Given2<String, String> {
   @override
   Future<void> executeStep(String commentType, String comment) async {
-    // TODO: implement executeStep
+    // implement executeStep
   }
 
   @override

lib/src/flutter/flutter_run_process_handler.dart

@@ -14,11 +14,18 @@ class FlutterRunProcessHandler extends ProcessHandler {
 
   static RegExp _noConnectedDeviceRegex =
       RegExp(r"no connected device", caseSensitive: false, multiLine: false);
+
+  static RegExp _restartedApplicationSuccessRegex = RegExp(
+      r"Restarted application (.*)ms.",
+      caseSensitive: false,
+      multiLine: false);
+
   Process _runningProcess;
   Stream<String> _processStdoutStream;
   List<StreamSubscription> _openSubscriptions = <StreamSubscription>[];
-  String _appTarget;
   String _workingDirectory;
+  String _appTarget;
+  bool _buildApp = true;
   String _buildFlavor;
   String _deviceTargetId;
 
@@ -38,13 +45,22 @@ class FlutterRunProcessHandler extends ProcessHandler {
     _deviceTargetId = deviceTargetId;
   }
 
+  void setBuildRequired(bool build) {
+    _buildApp = build;
+  }
+
   @override
   Future<void> run() async {
     final arguments = ["run", "--target=$_appTarget"];
 
+    if (_buildApp == false) {
+      arguments.add("--no-build");
+    }
+
     if (_buildFlavor.isNotEmpty) {
       arguments.add("--flavor=$_buildFlavor");
     }
+
     if (_deviceTargetId.isNotEmpty) {
       arguments.add("--device-id=$_deviceTargetId");
     }
@@ -75,28 +91,41 @@ class FlutterRunProcessHandler extends ProcessHandler {
     return exitCode;
   }
 
-  Future<String> waitForObservatoryDebuggerUri(
+  @override
-      [Duration timeout = const Duration(seconds: 60)]) {
+  Future restart() async {
+    _ensureRunningProcess();
+    _runningProcess.stdin.write("R");
+    return _waitForStdOutMessage(
+        _restartedApplicationSuccessRegex, "Timeout waiting for app restart");
+  }
+
+  Future<String> waitForObservatoryDebuggerUri() => _waitForStdOutMessage(
+      _observatoryDebuggerUriRegex,
+      "Timeout while waiting for observatory debugger uri");
+
+  Future<String> _waitForStdOutMessage(RegExp matcher, String timeoutMessage,
+      [Duration timeout = const Duration(seconds: 90)]) {
     _ensureRunningProcess();
     final completer = Completer<String>();
     StreamSubscription sub;
     sub = _processStdoutStream.timeout(timeout, onTimeout: (_) {
       sub?.cancel();
-      if (!completer.isCompleted)
+      if (!completer.isCompleted) {
-        completer.completeError(TimeoutException(
+        completer.completeError(TimeoutException(timeoutMessage, timeout));
-            "Timeout while wait for observatory debugger uri", timeout));
+      }
     }).listen((logLine) {
       // stdout.write(logLine);
-      if (_observatoryDebuggerUriRegex.hasMatch(logLine)) {
+      if (matcher.hasMatch(logLine)) {
         sub?.cancel();
-        if (!completer.isCompleted)
+        if (!completer.isCompleted) {
-          completer.complete(
+          completer.complete(matcher.firstMatch(logLine).group(1));
-              _observatoryDebuggerUriRegex.firstMatch(logLine).group(1));
+        }
       } else if (_noConnectedDeviceRegex.hasMatch(logLine)) {
         sub?.cancel();
-        if (!completer.isCompleted)
+        if (!completer.isCompleted) {
           stderr.writeln(
               "${FAIL_COLOR}No connected devices found to run app on and tests against$RESET_COLOR");
+        }
       }
     }, cancelOnError: true);
 

lib/src/flutter/flutter_test_configuration.dart

@@ -26,6 +26,10 @@ class FlutterTestConfiguration extends TestConfiguration {
   /// Defaults to empty
   String buildFlavor = "";
 
+  /// If the application should be built prior to running the tests
+  /// Defaults to true
+  bool build = true;
+
   /// The target device id to run the tests against when multiple devices detected
   /// Defaults to empty
   String targetDeviceId = "";
@@ -35,11 +39,11 @@ class FlutterTestConfiguration extends TestConfiguration {
   Future<FlutterDriver> createFlutterDriver([String dartVmServiceUrl]) async {
     dartVmServiceUrl = (dartVmServiceUrl ?? _observatoryDebuggerUri) ??
         Platform.environment['VM_SERVICE_URL'];
-    final driver = await FlutterDriver.connect(
+
+    return await FlutterDriver.connect(
         dartVmServiceUrl: dartVmServiceUrl,
         logCommunicationToFile: false,
         printCommunication: false);
-    return driver;
   }
 
   Future<FlutterWorld> createFlutterWorld(

lib/src/flutter/flutter_world.dart

@@ -11,7 +11,7 @@ class FlutterWorld extends World {
   }
 
   @override
-  void dispose() {
+  void dispose() async {
-    _driver.close();
+    await _driver?.close();
   }
 }

lib/src/flutter/hooks/app_runner_hook.dart

@@ -37,13 +37,15 @@ class FlutterAppRunnerHook extends Hook {
     haveRunFirstScenario = true;
     if (_flutterAppProcess != null &&
         flutterConfig.restartAppBetweenScenarios) {
-      await _terminateApp();
+      await _restartApp();
     }
   }
 
   Future<void> _runApp(FlutterTestConfiguration config) async {
     _flutterAppProcess = FlutterRunProcessHandler();
     _flutterAppProcess.setApplicationTargetFile(config.targetAppPath);
+    _flutterAppProcess
+        .setBuildRequired(haveRunFirstScenario ? false : config.build);
     _flutterAppProcess.setBuildFlavor(config.buildFlavor);
     _flutterAppProcess.setDeviceTargetId(config.targetDeviceId);
     stdout.writeln(
@@ -62,6 +64,16 @@ class FlutterAppRunnerHook extends Hook {
     }
   }
 
+  Future<void> _restartApp() async {
+    if (_flutterAppProcess != null) {
+      stdout.writeln("Restarting Flutter app under test");
+      await _flutterAppProcess.restart();
+      // it seems we need a small delay here otherwise the flutter driver fails to
+      // consistently connect
+      await Future.delayed(Duration(seconds: 1));
+    }
+  }
+
   FlutterTestConfiguration _castConfig(TestConfiguration config) =>
       config as FlutterTestConfiguration;
 }

pre-publish-checks.cmd

@@ -0,0 +1,3 @@
+CALL "C:\Google\flutter\bin\cache\dart-sdk\bin\dartanalyzer" --options analysis_options.yaml .
+CALL "C:\Google\flutter\bin\cache\dart-sdk\bin\dartfmt" . -w
+CALL flutter test

pubspec.lock

@@ -7,21 +7,21 @@ packages:
       name: analyzer
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.35.4"
+    version: "0.36.3"
   args:
     dependency: transitive
     description:
       name: args
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "1.5.1"
+    version: "1.5.2"
   async:
     dependency: transitive
     description:
       name: async
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "2.1.0"
+    version: "2.2.0"
   boolean_selector:
     dependency: transitive
     description:
@@ -57,13 +57,20 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.0.6"
+  csslib:
+    dependency: transitive
+    description:
+      name: csslib
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.16.0"
   file:
     dependency: transitive
     description:
       name: file
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "5.0.7"
+    version: "5.0.8"
   flutter:
     dependency: "direct main"
     description: flutter
@@ -85,7 +92,7 @@ packages:
       name: front_end
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.1.14"
+    version: "0.1.18"
   fuchsia_remote_debug_protocol:
     dependency: transitive
     description: flutter
@@ -97,7 +104,7 @@ packages:
       name: gherkin
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.0.4"
+    version: "1.0.0"
   glob:
     dependency: "direct main"
     description:
@@ -105,6 +112,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.1.7"
+  html:
+    dependency: transitive
+    description:
+      name: html
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.14.0+2"
   http:
     dependency: transitive
     description:
@@ -118,7 +132,7 @@ packages:
       name: http_multi_server
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "2.0.5"
+    version: "2.0.6"
   http_parser:
     dependency: transitive
     description:
@@ -153,14 +167,14 @@ packages:
       name: json_rpc_2
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "2.0.10"
+    version: "2.1.0"
   kernel:
     dependency: transitive
     description:
       name: kernel
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.3.14"
+    version: "0.3.18"
   matcher:
     dependency: "direct main"
     description:
@@ -181,7 +195,7 @@ packages:
       name: mime
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.9.6+2"
+    version: "0.9.6+3"
   multi_server_socket:
     dependency: transitive
     description:
@@ -223,7 +237,7 @@ packages:
       name: pedantic
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "1.5.0"
+    version: "1.7.0"
   platform:
     dependency: transitive
     description:
@@ -258,7 +272,7 @@ packages:
       name: quiver
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "2.0.2"
+    version: "2.0.3"
   shelf:
     dependency: transitive
     description:
@@ -347,21 +361,21 @@ packages:
       name: test
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "1.6.1"
+    version: "1.6.4"
   test_api:
     dependency: transitive
     description:
       name: test_api
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.2.4"
+    version: "0.2.6"
   test_core:
     dependency: transitive
     description:
       name: test_core
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.2.3"
+    version: "0.2.6"
   typed_data:
     dependency: transitive
     description:
@@ -382,7 +396,7 @@ packages:
       name: vm_service_client
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.2.6+1"
+    version: "0.2.6+2"
   watcher:
     dependency: transitive
     description:
@@ -396,7 +410,7 @@ packages:
       name: web_socket_channel
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "1.0.12"
+    version: "1.0.13"
   yaml:
     dependency: transitive
     description:

pubspec.yaml

@@ -1,6 +1,6 @@
 name: flutter_gherkin
 description: A Gherkin / Cucumber parser and test runner for Dart and Flutter
-version: 0.0.14
+version: 1.0.0
 author: Jon Samwell <jonsamwell@gmail.com>
 homepage: https://github.com/jonsamwell/flutter_gherkin
 
@@ -12,10 +12,10 @@ dependencies:
     sdk: flutter
   path: ^1.6.2
   glob: ^1.1.7
-  test: ^1.4.0
+  test: ^1.6.4
-  matcher: ^0.12.3+1
+  matcher: ^0.12.5
-  pedantic: ^1.4.0
+  pedantic: ^1.5.0
-  gherkin: ^0.0.4
+  gherkin: ^1.0.0
 
 dev_dependencies:
   flutter_test: