Merge branch 'wip-mdl-55986' of https://github.com/rajeshtaneja/moodle
authorEloy Lafuente (stronk7) <stronk7@moodle.org>
Tue, 4 Oct 2016 22:29:29 +0000 (00:29 +0200)
committerEloy Lafuente (stronk7) <stronk7@moodle.org>
Tue, 4 Oct 2016 22:29:29 +0000 (00:29 +0200)
21 files changed:
admin/tool/behat/cli/init.php
admin/tool/behat/cli/run.php
admin/tool/behat/cli/util.php
admin/tool/behat/cli/util_single_run.php
admin/tool/behat/tests/fixtures/core/behat_test_context_1.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/core/behat_test_context_2.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/core/test_1.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/core/test_2.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_behat_test_context_2.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_test_context_1.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_behat_test_context_1.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_test_context_2.php [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_1.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_2.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_3.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_4.feature [new file with mode: 0644]
admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_5.feature [new file with mode: 0644]
admin/tool/behat/tests/manager_test.php
admin/tool/behat/tests/manager_util_test.php
lib/behat/classes/behat_config_util.php
lib/behat/classes/util.php

index f58a1bc..37e1dc2 100644 (file)
@@ -48,6 +48,7 @@ list($options, $unrecognized) = cli_get_params(
         'fromrun'  => 1,
         'torun'    => 0,
         'run-with-theme' => false,
+        'optimize-runs' => '',
     ),
     array(
         'j' => 'parallel',
@@ -68,6 +69,7 @@ Options:
 -m, --maxruns    Max parallel processes to be executed at one time.
 --fromrun        Execute run starting from (Used for parallel runs on different vms)
 --torun          Execute run till (Used for parallel runs on different vms)
+--optimize-runs  Split features with specified tags in all parallel runs.
 --run-with-theme Run all core features with specified theme.
 
 -h, --help     Print out this help
@@ -85,21 +87,17 @@ if (!empty($options['help'])) {
 
 // Check which util file to call.
 $utilfile = 'util_single_run.php';
-$paralleloption = "";
+$commandoptions = "";
 // If parallel run then use utilparallel.
 if ($options['parallel'] && $options['parallel'] > 1) {
     $utilfile = 'util.php';
-    $paralleloption = "";
-    foreach ($options as $option => $value) {
-        if ($value) {
-            $paralleloption .= " --$option=\"$value\"";
-        }
-    }
 }
 
-$themesuitewithallfeatures = '';
-if ($options['run-with-theme']) {
-    $themesuitewithallfeatures = '--run-with-theme="true"';
+// Sanitize input options, so they can be passed to util.
+foreach ($options as $option => $value) {
+    if ($value) {
+        $commandoptions .= " --$option='$value'";
+    }
 }
 
 // Changing the cwd to admin/tool/behat/cli.
@@ -111,7 +109,7 @@ testing_update_composer_dependencies();
 
 // Check whether the behat test environment needs to be updated.
 chdir(__DIR__);
-exec("php $utilfile --diag $paralleloption $themesuitewithallfeatures", $output, $code);
+exec("php $utilfile --diag $commandoptions", $output, $code);
 
 if ($code == 0) {
     echo "Behat test environment already installed\n";
@@ -119,7 +117,7 @@ if ($code == 0) {
 } else if ($code == BEHAT_EXITCODE_INSTALL) {
     // Behat and dependencies are installed and we need to install the test site.
     chdir(__DIR__);
-    passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code);
+    passthru("php $utilfile --install $commandoptions", $code);
     if ($code != 0) {
         chdir($cwd);
         exit($code);
@@ -128,14 +126,14 @@ if ($code == 0) {
 } else if ($code == BEHAT_EXITCODE_REINSTALL) {
     // Test site data is outdated.
     chdir(__DIR__);
-    passthru("php $utilfile --drop $paralleloption $themesuitewithallfeatures", $code);
+    passthru("php $utilfile --drop $commandoptions", $code);
     if ($code != 0) {
         chdir($cwd);
         exit($code);
     }
 
     chdir(__DIR__);
-    passthru("php $utilfile --install $paralleloption $themesuitewithallfeatures", $code);
+    passthru("php $utilfile --install $commandoptions", $code);
     if ($code != 0) {
         chdir($cwd);
         exit($code);
@@ -150,7 +148,7 @@ if ($code == 0) {
 
 // Enable editing mode according to config.php vars.
 chdir(__DIR__);
-passthru("php $utilfile --enable $paralleloption $themesuitewithallfeatures", $code);
+passthru("php $utilfile --enable $commandoptions", $code);
 if ($code != 0) {
     echo "Error enabling site" . PHP_EOL;
     chdir($cwd);
index 853a17f..2775a5f 100644 (file)
@@ -55,7 +55,6 @@ list($options, $unrecognised) = cli_get_params(
         'fromrun'  => 1,
         'torun'    => 0,
         'single-run' => false,
-        'run-with-theme' => false,
     ),
     array(
         'h' => 'help',
@@ -79,7 +78,6 @@ Options:
 --replace          Replace args string with run process number, useful for output.
 --fromrun          Execute run starting from (Used for parallel runs on different vms)
 --torun            Execute run till (Used for parallel runs on different vms)
---run-with-theme   Run all core features with specified theme.
 
 -h, --help         Print out this help
 
@@ -120,8 +118,6 @@ array_walk($unrecognised, function (&$v) {
 });
 $extraopts = $unrecognised;
 
-$tags = '';
-
 if ($options['profile']) {
     $profile = $options['profile'];
 
@@ -174,33 +170,6 @@ if (empty($parallelrun)) {
     exit($code);
 }
 
-// Update config file if tags defined.
-if ($tags) {
-    define('ABORT_AFTER_CONFIG_CANCEL', true);
-    require("$CFG->dirroot/lib/setup.php");
-    // Hack to set proper dataroot and wwwroot.
-    $behatdataroot = $CFG->behat_dataroot;
-    $behatwwwroot  = $CFG->behat_wwwroot;
-    for ($i = 1; $i <= $parallelrun; $i++) {
-        $CFG->behatrunprocess = $i;
-
-        if (!empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
-            $CFG->behat_wwwroot = $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'];
-        } else {
-            $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i;
-        }
-        if (!empty($CFG->behat_parallel_run[$i - 1]['behat_dataroot'])) {
-            $CFG->behat_dataroot = $CFG->behat_parallel_run[$i - 1]['behat_dataroot'];
-        } else {
-            $CFG->behat_dataroot = $behatdataroot . $i;
-        }
-        behat_config_manager::update_config_file('', true, $tags, $options['run-with-theme'], $parallelrun);
-    }
-    $CFG->behat_dataroot = $behatdataroot;
-    $CFG->behat_wwwroot = $behatwwwroot;
-    unset($CFG->behatrunprocess);
-}
-
 $cmds = array();
 echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
 
index a30e30a..e82e7fb 100644 (file)
@@ -57,6 +57,7 @@ list($options, $unrecognized) = cli_get_params(
         'fromrun'     => 1,
         'torun'       => 0,
         'run-with-theme' => false,
+        'optimize-runs' => '',
     ),
     array(
         'h' => 'help',
@@ -80,8 +81,9 @@ Options:
 --diag         Get behat test environment status code
 --updatesteps  Update feature step file.
 
--j, --parallel Number of parallel behat run operation
--m, --maxruns  Max parallel processes to be executed at one time.
+-j, --parallel   Number of parallel behat run operation
+-m, --maxruns    Max parallel processes to be executed at one time.
+--optimize-runs  Split features with specified tags in all parallel runs.
 --run-with-theme Run all core features with specified theme.
 
 -h, --help     Print out this help
@@ -185,7 +187,7 @@ if ($options['diag'] || $options['enable'] || $options['disable']) {
             $CFG->behatrunprocess = $i;
 
             // Update config file for each run.
-            behat_config_manager::update_config_file('', true, '', $options['run-with-theme'],
+            behat_config_manager::update_config_file('', true, $options['optimize-runs'], $options['run-with-theme'],
                 $options['parallel'], $i);
         }
         unset($CFG->behatrunprocess);
@@ -267,7 +269,7 @@ function commands_to_execute($options) {
         if ($options[$option]) {
             $extra .= " --$option";
             if ($value) {
-                $extra .= "=$value";
+                $extra .= "='$value'";
             }
         }
     }
index 70aca44..85987a0 100644 (file)
@@ -48,6 +48,7 @@ list($options, $unrecognized) = cli_get_params(
         'tags'        => '',
         'updatesteps' => false,
         'run-with-theme' => false,
+        'optimize-runs' => '',
     ),
     array(
         'h' => 'help',
@@ -72,6 +73,7 @@ Options:
 --disable        Disables test environment
 --diag           Get behat test environment status code
 --updatesteps    Update feature step file.
+--optimize-runs  Split features with specified tags in all parallel runs.
 --run-with-theme Run all core features with specified theme.
 
 -h, --help Print out this help
@@ -178,7 +180,7 @@ if ($options['install']) {
     }
 
     // Enable test mode.
-    behat_util::start_test_mode($options['run-with-theme'], $parallel, $run);
+    behat_util::start_test_mode($options['run-with-theme'], $options['optimize-runs'], $parallel, $run);
 
     // This is only displayed once for parallel install.
     if (empty($run)) {
diff --git a/admin/tool/behat/tests/fixtures/core/behat_test_context_1.php b/admin/tool/behat/tests/fixtures/core/behat_test_context_1.php
new file mode 100644 (file)
index 0000000..01c0396
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test context 1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../../../../../lib/behat/behat_base.php');
+
+/**
+ * Test context 1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_test_context_1 extends behat_base {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/core/behat_test_context_2.php b/admin/tool/behat/tests/fixtures/core/behat_test_context_2.php
new file mode 100644 (file)
index 0000000..0dbdbeb
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../../../../../lib/behat/behat_base.php');
+
+/**
+ * Test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_test_context_2 extends behat_base {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/core/test_1.feature b/admin/tool/behat/tests/fixtures/core/test_1.feature
new file mode 100644 (file)
index 0000000..984f6e7
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test1
+Feature: Test feature 1
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/core/test_2.feature b/admin/tool/behat/tests/fixtures/core/test_2.feature
new file mode 100644 (file)
index 0000000..091781e
--- /dev/null
@@ -0,0 +1,8 @@
+@behat_test @test2
+Feature: Test feature 2
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_behat_test_context_2.php b/admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_behat_test_context_2.php
new file mode 100644 (file)
index 0000000..710da0a
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Theme test context 1 overriding test_1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../behat_test_context_1.php');
+
+/**
+ * Theme test context 1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_theme_nofeatures_behat_test_context_1 extends behat_test_context_1 {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_test_context_1.php b/admin/tool/behat/tests/fixtures/theme/nofeatures/behat_theme_nofeatures_test_context_1.php
new file mode 100644 (file)
index 0000000..87fe40c
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Theme test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../../../../../lib/behat/behat_base.php');
+
+/**
+ * Theme test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_theme_nofeatures_test_context_2 extends behat_base {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_behat_test_context_1.php b/admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_behat_test_context_1.php
new file mode 100644 (file)
index 0000000..3941071
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Theme test context 1 overriding test_1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../behat_test_context_1.php');
+
+/**
+ * Theme test context 1
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_theme_withfeatures_behat_test_context_1 extends behat_test_context_1 {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_test_context_2.php b/admin/tool/behat/tests/fixtures/theme/withfeatures/behat_theme_withfeatures_test_context_2.php
new file mode 100644 (file)
index 0000000..23eedff
--- /dev/null
@@ -0,0 +1,38 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Theme test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+// NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
+
+require_once(__DIR__ . '/../../../../../lib/behat/behat_base.php');
+
+/**
+ * Theme test context 2
+ *
+ * @package    tool_behat
+ * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class behat_theme_withfeatures_test_context_2 extends behat_base {
+
+}
\ No newline at end of file
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_1.feature b/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_1.feature
new file mode 100644 (file)
index 0000000..ff72285
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test1 @testtheme @commontag
+Feature: Test feature 1
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_2.feature b/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_2.feature
new file mode 100644 (file)
index 0000000..7e913c6
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test2 @testtheme @commontag
+Feature: Test feature 2
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_3.feature b/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_3.feature
new file mode 100644 (file)
index 0000000..d3ea3d9
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test3 @testtheme
+Feature: Test feature 3
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_4.feature b/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_4.feature
new file mode 100644 (file)
index 0000000..3825f9f
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test4 @testtheme
+Feature: Test feature 4
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
diff --git a/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_5.feature b/admin/tool/behat/tests/fixtures/theme/withfeatures/theme_test_5.feature
new file mode 100644 (file)
index 0000000..7476252
--- /dev/null
@@ -0,0 +1,9 @@
+@behat_test @test5 @testtheme
+Feature: Test feature 5
+  In order to test behat.yml in phpunit
+  As an user
+  I need to be able to include this feature
+
+  @javascript
+  Scenario: I should not be included in normal execution.
+    Given I pause scenario execution
index ae052e5..7b88c80 100644 (file)
@@ -114,6 +114,8 @@ class tool_behat_manager_testcase extends advanced_testcase {
     public function test_config_file_contents() {
         global $CFG;
 
+        $this->resetAfterTest(true);
+
         // Skip tests if behat is not installed.
         $vendorpath = $CFG->dirroot . '/vendor';
         if (!file_exists($vendorpath . '/autoload.php') || !is_dir($vendorpath . '/behat')) {
@@ -145,17 +147,19 @@ class tool_behat_manager_testcase extends advanced_testcase {
         // YAML decides when is is necessary to wrap strings between single quotes, so not controlled
         // values like paths should not be asserted including the key name as they would depend on the
         // directories values.
-        $this->assertContains($CFG->dirroot, $contents);
+        $this->assertContains($CFG->dirroot,
+            $contents['default']['extensions']['Moodle\BehatExtension']['moodledirroot']);
 
         // Not quoted strings.
-        $this->assertContains('micarro: /me/lo/robaron', $contents);
+        $this->assertEquals('/me/lo/robaron',
+            $contents['default']['extensions']['Moodle\BehatExtension']['steps_definitions']['micarro']);
 
         // YAML uses single quotes to wrap URL strings.
-        $this->assertContains("base_url: '" . $CFG->behat_wwwroot . "'", $contents);
+        $this->assertEquals($CFG->behat_wwwroot, $contents['default']['extensions']['Behat\MinkExtension']['base_url']);
 
         // Lists.
-        $this->assertContains('- feature1', $contents);
-        $this->assertContains('- feature3', $contents);
+        $this->assertEquals('feature1', $contents['default']['suites']['default']['paths'][0]);
+        $this->assertEquals('feature3', $contents['default']['suites']['default']['paths'][2]);
 
         unset($CFG->behat_wwwroot);
     }
index 9556cee..5723743 100644 (file)
@@ -41,314 +41,385 @@ require_once($CFG->libdir . '/behat/classes/behat_config_manager.php');
  */
 class tool_behat_manager_util_testcase extends advanced_testcase {
 
-    /**
-     * @var array core features.
-     */
-    private $corefeatures = array(
-        'feedback_editpdf_behat_test1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-        'feedback_file_behat_test2' => "C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature",
-        'moodle_login_behat_test3' => "C:\\test\\moodle/login/tests/behat/behat_test3.feature",
-        );
-
+    /** @var array Fixtures features which are available. */
+    private $featurepaths = array(
+        'default' => array(
+            'test_1.feature',
+            'test_2.feature',
+        ),
+        'withfeatures' => array(
+            'theme_test_1.feature',
+            'theme_test_2.feature',
+            'theme_test_3.feature',
+            'theme_test_4.feature',
+            'theme_test_5.feature',
+        ),
+        'nofeatures' => array()
+    );
+
+    /** @var array Fixture contexts which are available */
+    private $contextspath = array(
+        'default' => array(
+            'behat_test_context_1',
+            'behat_test_context_2'
+        ),
+        'withfeatures' => array(
+            'behat_test_context_2',
+            'behat_theme_withfeatures_test_context_2',
+            'behat_theme_withfeatures_behat_test_context_1'
+        ),
+        'nofeatures' => array(
+            'behat_test_context_1',
+            'behat_theme_nofeatures_test_context_1',
+            'behat_theme_nofeatures_behat_test_context_2'
+        ),
+    );
+
+    private $corefatures = array('test_1' => __DIR__.'/fixtures/core/test_1.feature',
+                                 'test_2' => __DIR__.'/fixtures/core/test_2.feature');
+
+    private $corecontexts = array('behat_test_context_1' => __DIR__.'/fixtures/core/behat_test_context_1.php',
+                                  'behat_test_context_2' => __DIR__.'/fixtures/core/behat_test_context_2.php');
 
     /**
-     * @var array theme features.
+     * Setup test.
      */
-    private $themefeatures = array(
-            'behat_themetest1_core_behat_tests_testtheme_theme' => '/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature',
-            'behat_themetest2_mod_assign_behat_tests_testtheme_theme' => "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature",
-            'behat_themetest3_behat_tests_testtheme_theme_moodle' => "C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature",
-        );
-
-    /**
-     * @var array core contexts.
-     */
-    private $corecontexts = array(
-            'behat_context1' => '/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php',
-            'behat_context2' => "C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php",
-            'behat_context3' => "C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php",
-        );
+    public function setup() {
+        global $CFG;
 
-    /**
-     * @var array Theme contexts for test.
-     */
-    private $themecontexts = array(
-            'behat_theme_testtheme_behat_context1' =>
-                '/test/moodle/theme/testtheme/tests/behat/mod_assign/behat_theme_testtheme_behat_context1.php',
-            'behat_theme_testtheme_behat_context2' =>
-                "C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\block_comments\\behat_theme_testtheme_behat_context2.php",
-            'behat_theme_testtheme_behat_context3' =>
-                "C:\\test\\moodle/theme/testtheme/tests/behat/editor_atto/behat_theme_testtheme_behat_context3.php"
-        );
+        $this->resetAfterTest();
+        $CFG->behat_wwwroot = 'http://example.com/behat';
+    }
 
     /**
-     * Keep instance of behat_config_util mock object.
+     * Utility function to build mock object.
      *
-     * @var null
-     */
-    private $behatconfigutil = null;
-
-    /**
-     * Test setup.
+     * @param  behat_config_util $behatconfigutil
+     * @param bool $notheme
+     * @return mixed
      */
-    public function setUp() {
-        $this->resetAfterTest(true);
-        $mockbuilder = $this->getMockBuilder('behat_config_util');
-        $mockbuilder->setMethods(array('get_behat_features_for_theme', 'get_behat_contexts_for_theme',
-            'get_list_of_themes', 'get_overridden_theme_contexts'));
-
-        $this->behatconfigutil = $mockbuilder->getMock();
+    private function get_behat_config_util($behatconfigutil, $notheme = false) {
+        // Create a map of arguments to return values.
+        $map = array(
+            array('withfeatures', __DIR__.'/fixtures/theme/withfeatures'),
+            array('nofeatures', __DIR__.'/fixtures/theme/nofeatures')
+        );
 
         // List of themes is const for test.
-        $this->behatconfigutil->expects($this->any())
+        if ($notheme) {
+            $themelist = array();
+        } else {
+            $themelist = array('withfeatures', 'nofeatures');
+        }
+
+        $behatconfigutil->expects($this->any())
             ->method('get_list_of_themes')
-            ->will($this->returnValue(array('testtheme')));
+            ->will($this->returnValue($themelist));
 
-        $this->behatconfigutil->expects($this->any())
-            ->method('get_behat_contexts_for_theme')
-            ->with($this->equalTo('testtheme'))
-            ->will($this->returnValue(array(array_keys($this->themecontexts), $this->themecontexts)));
+        // Theme directory for testing.
+        $behatconfigutil->expects($this->any())
+            ->method('get_theme_test_directory')
+            ->will($this->returnValueMap($map));
 
+        return $behatconfigutil;
     }
 
     /**
      * Behat config for single run.
-     *
      */
     public function test_get_config_file_contents_with_single_run() {
-        global $CFG;
 
-        $CFG->behat_wwwroot = 'http://example.com/behat';
+        $mockbuilder = $this->getMockBuilder('behat_config_util');
+        $mockbuilder->setMethods(array('get_theme_test_directory', 'get_list_of_themes'));
 
-        $behatconfigutil = $this->behatconfigutil;
-
-        // No theme feature exists.
-        $behatconfigutil->expects($this->once())
-            ->method('get_behat_features_for_theme')
-            ->with($this->anything())
-            ->will($this->returnValue(array(array(), array())));
-
-        $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts);
-
-        $expectedconfigwithfeatures = "default:
-  formatters:
-    moodle_progress:
-      output_styles:
-        comment:
-          - magenta
-  suites:
-    default:
-      paths:
-        - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature
-        - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature'
-        - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature'
-      contexts:
-        - behat_context1
-        - behat_context2
-        - behat_context3
-    testtheme:
-      paths: {  }
-      contexts:
-        - behat_theme_testtheme_behat_context1
-        - behat_theme_testtheme_behat_context2
-        - behat_theme_testtheme_behat_context3
-  extensions:
-    Behat\\MinkExtension:
-      base_url: 'http://example.com/behat'
-      goutte: null
-      selenium2:
-        wd_host: 'http://localhost:4444/wd/hub'
-";
-
-        $this->assertContains($expectedconfigwithfeatures, $config);
-
-        $expectedstepdefinitions = "steps_definitions:
-        behat_context1: /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php
-        behat_context2: 'C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php'
-        behat_context3: 'C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php'
-";
-        $this->assertContains($expectedstepdefinitions, $config);
-    }
+        $behatconfigutil = $mockbuilder->getMock();
 
-    /**
-     * Behat config for parallel run.
-     */
-    public function test_get_config_file_contents_with_parallel_run() {
-        global $CFG;
+        $behatconfigutil = $this->get_behat_config_util($behatconfigutil);
+        $config = $behatconfigutil->get_config_file_contents($this->corefatures, $this->corecontexts);
 
-        $CFG->behat_wwwroot = 'http://example.com/behat';
-        $behatconfigutil = $this->behatconfigutil;
+        // Two suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
 
-        // No theme feature exists.
-        $behatconfigutil->expects($this->any())
-            ->method('get_behat_features_for_theme')
-            ->with($this->anything())
-            ->will($this->returnValue(array(array(), array())));
-
-        $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 1);
-
-        // First run.
-        $this->assertContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature',
-            $config);
-
-        // Second run.
-        $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 2);
-
-        $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-            $config);
-        $this->assertContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature',
-            $config);
-
-        $config = $behatconfigutil->get_config_file_contents($this->corefeatures, $this->corecontexts, '', 3, 3);
-
-        $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature',
-            $config);
-        $this->assertContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature',
-            $config);
+        // Check features.
+        foreach ($this->featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
     }
 
     /**
-     * Behat config with theme features.
+     * Behat config for single run with no theme installed.
      */
-    public function test_get_config_file_contents_with_theme_features() {
-        global $CFG;
+    public function test_get_config_file_contents_with_single_run_no_theme() {
 
-        $behatconfigutil = $this->behatconfigutil;
+        $mockbuilder = $this->getMockBuilder('behat_config_util');
+        $mockbuilder->setMethods(array('get_theme_test_directory', 'get_list_of_themes'));
 
-        $suitefeatures = array_merge($this->corefeatures, $this->themefeatures);
-        $themefeatures = $this->themefeatures;
-        $behatconfigutil->expects($this->once())
-            ->method('get_behat_features_for_theme')
-            ->with($this->equalTo('testtheme'))
-            ->will($this->returnValue(array(array(), $themefeatures)));
+        $behatconfigutil = $mockbuilder->getMock();
 
-        $behatconfigutil->expects($this->once())
-            ->method('get_behat_contexts_for_theme')
-            ->with($this->equalTo('testtheme'))
-            ->will($this->returnValue(array(array_keys($this->themecontexts), $this->themecontexts)));
+        $behatconfigutil = $this->get_behat_config_util($behatconfigutil, true);
+        $config = $behatconfigutil->get_config_file_contents($this->corefatures, $this->corecontexts);
 
-        $behatconfigutil->expects($this->once())
-            ->method('get_overridden_theme_contexts')
-            ->will($this->returnValue($this->themecontexts));
-        $behatconfigutil->set_theme_suite_to_include_core_features(true);
+        // Two suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(1, $suites);
 
-        $CFG->behat_wwwroot = 'http://example.com/behat';
-        $config = $behatconfigutil->get_config_file_contents($suitefeatures, $this->corecontexts);
-
-        $expectedconfigwithfeatures = "default:
-  formatters:
-    moodle_progress:
-      output_styles:
-        comment:
-          - magenta
-  suites:
-    default:
-      paths:
-        - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature
-        - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature'
-        - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature'
-      contexts:
-        - behat_context1
-        - behat_context2
-        - behat_context3
-    testtheme:
-      paths:
-        - /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature
-        - 'C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature'
-        - 'C:\\test\\moodle/login/tests/behat/behat_test3.feature'
-        - /test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature
-        - 'C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature'
-        - 'C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature'
-      contexts:
-        - behat_theme_testtheme_behat_context1
-        - behat_theme_testtheme_behat_context2
-        - behat_theme_testtheme_behat_context3
-  extensions:
-    Behat\\MinkExtension:
-      base_url: 'http://example.com/behat'
-      goutte: null
-      selenium2:
-        wd_host: 'http://localhost:4444/wd/hub'
-";
-        $this->assertContains($expectedconfigwithfeatures, $config);
-
-        $expectedstepdefinitions = "steps_definitions:
-        behat_context1: /test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_context1.php
-        behat_context2: 'C:\\test\\moodle\\blocks\\comments\\tests\\behat\\behat_context2.php'
-        behat_context3: 'C:\\test\\moodle/lib/editor/atto/tests/behat/behat_context3.php'
-        behat_theme_testtheme_behat_context1: /test/moodle/theme/testtheme/tests/behat/mod_assign/behat_theme_testtheme_behat_context1.php
-        behat_theme_testtheme_behat_context2: 'C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\block_comments\\behat_theme_testtheme_behat_context2.php'
-        behat_theme_testtheme_behat_context3: 'C:\\test\\moodle/theme/testtheme/tests/behat/editor_atto/behat_theme_testtheme_behat_context3.php'";
-
-        $this->assertContains($expectedstepdefinitions, $config);
+        $featurepaths = array(
+            'default' => array(
+                'test_1.feature',
+                'test_2.feature',
+            )
+        );
+
+        $contextspath = array(
+            'default' => array(
+                'behat_test_context_1',
+                'behat_test_context_2'
+            )
+        );
+
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+
+        // Check contexts.
+        foreach ($contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+
+        // There are 6 step definitions.
+        $this->assertCount(2, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
     }
 
     /**
      * Behat config for parallel run.
      */
-    public function test_get_config_file_contents_with_theme_and_parallel_run() {
-        global $CFG;
+    public function test_get_config_file_contents_with_parallel_run() {
 
-        $CFG->behat_wwwroot = 'http://example.com/behat';
+        $mockbuilder = $this->getMockBuilder('behat_config_util');
+        $mockbuilder->setMethods(array('get_theme_test_directory', 'get_list_of_themes'));
+
+        $behatconfigutil = $mockbuilder->getMock();
+
+        $behatconfigutil = $this->get_behat_config_util($behatconfigutil);
+
+        // Test first run out of 3.
+        $config = $behatconfigutil->get_config_file_contents($this->corefatures, $this->corecontexts, '', 3, 1);
+        // Three suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is first feature file in first run.
+        $featurepaths = array(
+            'default' => array('test_1.feature'),
+            'withfeatures' => array('theme_test_1.feature', 'theme_test_2.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
+
+        // Test second run out of 3.
+        $config = $behatconfigutil->get_config_file_contents('', '', '', 3, 2);
+        // Three suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is second feature file in first run.
+        $featurepaths = array(
+            'default' => array('test_2.feature'),
+            'withfeatures' => array('theme_test_3.feature', 'theme_test_4.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
+
+        // Test third run out of 3.
+        $config = $behatconfigutil->get_config_file_contents('', '', '', 3, 3);
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is second feature file in first run.
+        $featurepaths = array(
+            'default' => array(),
+            'withfeatures' => array('theme_test_5.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
+    }
 
-        $behatconfigutil = $this->behatconfigutil;
+    /**
+     * Behat config for parallel run.
+     */
+    public function test_get_config_file_contents_with_parallel_run_optimize_tags() {
+
+        $mockbuilder = $this->getMockBuilder('behat_config_util');
+        $mockbuilder->setMethods(array('get_theme_test_directory', 'get_list_of_themes'));
 
-        $features = array_merge($this->corefeatures, $this->themefeatures);
-        $themefeatures = $this->themefeatures;
-        $behatconfigutil->expects($this->atLeastOnce())
-            ->method('get_behat_features_for_theme')
-            ->with($this->equalTo('testtheme'))
-            ->will($this->returnValue(array(array(), $themefeatures)));
+        $behatconfigutil = $mockbuilder->getMock();
 
-        $behatconfigutil->expects($this->atLeastOnce())
-            ->method('get_behat_contexts_for_theme')
-            ->with($this->equalTo('testtheme'))
-            ->will($this->returnValue(array(array_keys($this->themecontexts), $this->themecontexts)));
+        $behatconfigutil = $this->get_behat_config_util($behatconfigutil);
 
-        $CFG->behat_wwwroot = 'http://example.com/behat';
+        // Test first run out of 3.
+        $config = $behatconfigutil->get_config_file_contents($this->corefatures, $this->corecontexts, '@commontag', 3, 1);
 
-        $behatconfigutil->set_theme_suite_to_include_core_features(false);
-
-        $config = $behatconfigutil->get_config_file_contents($features, $this->themecontexts, '', 3, 1);
-
-        // First run.
-        $this->assertContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature',
-            $config);
-        // Theme suite features.
-        $this->assertContains('/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature',
-            $config);
-
-        // Second run.
-        $config = $behatconfigutil->get_config_file_contents($features, $this->themecontexts, '', 3, 2);
-        $this->assertNotContains('/test/moodle/mod/assign/feedback/editpdf/tests/behat/behat_test1.feature',
-            $config);
-        $this->assertContains('C:\\test\\moodle\\mod\\assign\\feedback\\file\\tests\\behat\\behat_test2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/login/tests/behat/behat_test3.feature',
-            $config);
-        // Theme suite features.
-        $this->assertNotContains('/test/moodle/theme/testtheme/tests/behat/core/behat_themetest1.feature',
-            $config);
-        $this->assertContains('C:\\test\\moodle\\theme\\testtheme\\tests\\behat\\mod_assign\\behat_themetest2.feature',
-            $config);
-        $this->assertNotContains('C:\\test\\moodle/theme/testtheme/tests/behat/behat_themetest3.feature',
-            $config);
+        // Three suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is first feature file in first run.
+        $featurepaths = array(
+            'default' => array('test_1.feature'),
+            'withfeatures' => array('theme_test_1.feature', 'theme_test_3.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
+
+        // Test second run out of 3.
+        $config = $behatconfigutil->get_config_file_contents('', '', '@commontag', 3, 2);
+
+        // Three suites should be present.
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is second feature file in first run.
+        $featurepaths = array(
+            'default' => array('test_2.feature'),
+            'withfeatures' => array('theme_test_2.feature', 'theme_test_4.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
+
+        // Test third run out of 3.
+        $config = $behatconfigutil->get_config_file_contents('', '', '', 3, 3);
+        $suites = $config['default']['suites'];
+        $this->assertCount(3, $suites);
+        // There is second feature file in first run.
+        $featurepaths = array(
+            'default' => array(),
+            'withfeatures' => array('theme_test_5.feature'),
+            'nofeatures' => array()
+        );
+        // Check features.
+        foreach ($featurepaths as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['paths']);
+
+            foreach ($paths as $key => $feature) {
+                $this->assertContains($feature, $suites[$themename]['paths'][$key]);
+            }
+        }
+        // Check contexts.
+        foreach ($this->contextspath as $themename => $paths) {
+            $this->assertCount(count($paths), $suites[$themename]['contexts']);
+
+            foreach ($paths as $key => $context) {
+                $this->assertTrue(in_array($context, $suites[$themename]['contexts']));
+            }
+        }
+        // There are 6 step definitions.
+        $this->assertCount(6, $config['default']['extensions']['Moodle\BehatExtension']['steps_definitions']);
     }
 
     /**
@@ -363,7 +434,8 @@ class tool_behat_manager_util_testcase extends advanced_testcase {
         $oldroot = $CFG->dirroot;
         $CFG->dirroot = 'C:';
 
-        $behatconfigutil = $this->behatconfigutil;
+        $behatconfigutil = new behat_config_util();
+
         // Fix expected directory path for OS.
         $cleanfeaturepath = testing_cli_fix_directory_separator($cleanfeaturepath);
 
index ba3bced..e07b512 100644 (file)
@@ -210,7 +210,15 @@ class behat_config_util {
 
         $this->features = $features;
 
-        return $this->filtered_features_with_tags($features, $tags);
+        // If tags are passed then filter features which has sepecified tags.
+        if (!empty($tags)) {
+            $features = $this->filtered_features_with_tags($features, $tags);
+        }
+
+        // Return sorted list.
+        ksort($features);
+
+        return $features;
     }
 
     /**
@@ -301,6 +309,9 @@ class behat_config_util {
             }
         }
 
+        // Sort contexts with there name.
+        ksort($this->contexts);
+
         return $this->get_component_contexts($component);
     }
 
@@ -351,6 +362,11 @@ class behat_config_util {
 
         $config = $this->merge_behat_profiles($config);
 
+        // Return config array for phpunit, so it can be tested.
+        if (defined('PHPUNIT_TEST') && PHPUNIT_TEST) {
+            return $config;
+        }
+
         return Symfony\Component\Yaml\Yaml::dump($config, 10, 2);
     }
 
@@ -390,7 +406,7 @@ class behat_config_util {
             $andtags[] = preg_replace('/,.*/', '', $tag);
         }
 
-        foreach ($features as $featurefile) {
+        foreach ($features as $key => $featurefile) {
             $contents = file_get_contents($featurefile);
             $includefeature = true;
             foreach ($andtags as $tag) {
@@ -418,7 +434,7 @@ class behat_config_util {
             }
 
             if ($includefeature) {
-                $newfeaturelist[] = $featurefile;
+                $newfeaturelist[$key] = $featurefile;
             }
         }
         return $newfeaturelist;
@@ -520,20 +536,31 @@ class behat_config_util {
 
         // If parallel run, then only divide features.
         if (!empty($currentrun) && !empty($parallelruns)) {
+
+            $featurestodivide['withtags'] = $features;
+            $allocatedfeatures = array();
+
+            // If tags are set then split features with tags first.
+            if (!empty($this->tags)) {
+                $featurestodivide['withtags'] = $this->filtered_features_with_tags($features);
+                $featurestodivide['withouttags'] = $this->remove_blacklisted_features_from_list($features,
+                    $featurestodivide['withtags']);
+            }
+
             // Attempt to split into weighted buckets using timing information, if available.
-            if ($alloc = $this->profile_guided_allocate($features, max(1, $parallelruns), $currentrun)) {
-                $allocatedfeatures = $alloc;
-            } else {
-                // Divide the list of feature files amongst the parallel runners.
-                // Pull out the features for just this worker.
-                if (count($features)) {
-                    $features = array_chunk($features, ceil(count($features) / max(1, $parallelruns)));
-
-                    // Check if there is any feature file for this process.
-                    if (!empty($features[$currentrun - 1])) {
-                        $allocatedfeatures = $features[$currentrun - 1];
-                    } else {
-                        $allocatedfeatures = array();
+            foreach ($featurestodivide as $tagfeatures) {
+                if ($alloc = $this->profile_guided_allocate($tagfeatures, max(1, $parallelruns), $currentrun)) {
+                    $allocatedfeatures = array_merge($allocatedfeatures, $alloc);
+                } else {
+                    // Divide the list of feature files amongst the parallel runners.
+                    // Pull out the features for just this worker.
+                    if (count($tagfeatures)) {
+                        $splitfeatures = array_chunk($tagfeatures, ceil(count($tagfeatures) / max(1, $parallelruns)));
+
+                        // Check if there is any feature file for this process.
+                        if (!empty($splitfeatures[$currentrun - 1])) {
+                            $allocatedfeatures = array_merge($allocatedfeatures, $splitfeatures[$currentrun - 1]);
+                        }
                     }
                 }
             }
@@ -863,7 +890,7 @@ class behat_config_util {
      *
      * @return array
      */
-    private function get_components_with_tests() {
+    protected function get_components_with_tests() {
         if (empty($this->componentswithtests)) {
             $this->componentswithtests = tests_finder::get_components_with_tests('behat');
         }
@@ -902,9 +929,7 @@ class behat_config_util {
             if (isset($features[$key])) {
                 $features[$key] = null;
                 unset($features[$key]);
-            } else if (empty($this->tags)) {
-                // If tags not set, then ensure we have a blacklisted feature in core. Else, let user know that
-                // blacklisted feature is invalid.
+            } else {
                 $featurestocheck = $this->get_components_features();
                 if (!isset($featurestocheck[$key]) && !defined('PHPUNIT_TEST')) {
                     behat_error(BEHAT_EXITCODE_REQUIREMENT, 'Blacklisted feature "' . $blacklistpath . '" not found.');
@@ -935,9 +960,7 @@ class behat_config_util {
             $currentrun = $this->get_current_run();;
         }
 
-        $blacklistedfeatures = array();
         $themefeatures = array();
-        $themesuitecontexts = array();
         $themecontexts = array();
 
         $themes = $this->get_list_of_themes();
@@ -945,34 +968,35 @@ class behat_config_util {
         // Create list of theme suite features and contexts.
         foreach ($themes as $theme) {
             // Get theme features.
-            list($blacklistedfeatures[$theme], $themefeatures[$theme]) = $this->get_behat_features_for_theme($theme);
+            $themefeatures[$theme] = $this->get_behat_features_for_theme($theme);
 
-            list($themecontexts[$theme], $themesuitecontexts[$theme]) = $this->get_behat_contexts_for_theme($theme);
+            $themecontexts[$theme] = $this->get_behat_contexts_for_theme($theme);
         }
 
         // Remove list of theme features for default suite, as default suite should not run theme specific features.
         foreach ($themefeatures as $removethemefeatures) {
-            $features = $this->remove_blacklisted_features_from_list($features, $removethemefeatures);
+            if (!empty($removethemefeatures['features'])) {
+                $features = $this->remove_blacklisted_features_from_list($features, $removethemefeatures['features']);
+            }
         }
 
         // Remove list of theme features for default suite, as default suite should not run theme specific features.
-        foreach ($themecontexts as $theme => $themeblacklistcontexts) {
-            if ($themeblacklistcontexts) {
-                foreach ($themeblacklistcontexts as $c) {
+        foreach ($themecontexts as $themename => $themecontext) {
+            if (!empty($themecontext['contexts'])) {
+                foreach ($themecontext['contexts'] as $contextkey => $contextpath) {
                     // Remove theme specific contexts from default contexts.
-                    unset($contexts[$c]);
+                    unset($contexts[$contextkey]);
 
                     // Remove theme specific contexts from other themes.
                     foreach ($themes as $currenttheme) {
-                        if (($currenttheme != $theme) && isset($themesuitecontexts[$currenttheme][$c])) {
-                            unset($themesuitecontexts[$currenttheme][$c]);
+                        if (($currenttheme != $themename) && isset($themecontexts[$currenttheme]['suitecontexts'][$contextkey])) {
+                            unset($themecontexts[$currenttheme]['suitecontexts'][$contextkey]);
                         }
                     }
                 }
             }
         }
 
-        // Return sub-set of features if parallel run.
         $featuresforrun = $this->get_features_for_the_run($features, $parallelruns, $currentrun);
 
         // Default suite.
@@ -989,17 +1013,18 @@ class behat_config_util {
             // If theme suite with all features is set, then we want all core features to be part of theme suite.
             if ($this->themesuitewithallfeatures) {
                 // If there is no theme specific feature. Then it's just core features.
-                if (empty($themefeatures[$theme])) {
+                if (empty($themefeatures[$theme]['features'])) {
                     $themesuitefeatures = $features;
                 } else {
-                    $themesuitefeatures = array_merge($features, $themefeatures[$theme]);
+                    $themesuitefeatures = array_merge($features, $themefeatures[$theme]['features']);
                 }
             } else {
-                $themesuitefeatures = $themefeatures[$theme];
+                $themesuitefeatures = $themefeatures[$theme]['features'];
             }
 
             // Remove blacklisted features.
-            $themesuitefeatures = $this->remove_blacklisted_features_from_list($themesuitefeatures, $blacklistedfeatures[$theme]);
+            $themesuitefeatures = $this->remove_blacklisted_features_from_list($themesuitefeatures,
+                $themefeatures[$theme]['blacklistfeatures']);
 
             // Return sub-set of features if parallel run.
             $themesuitefeatures = $this->get_features_for_the_run($themesuitefeatures, $parallelruns, $currentrun);
@@ -1009,7 +1034,7 @@ class behat_config_util {
             $suites = array_merge($suites, array(
                 $theme => array(
                     'paths'    => array_values($themesuitefeatures),
-                    'contexts' => array_keys($themesuitecontexts[$theme]),
+                    'contexts' => array_keys($themecontexts[$theme]['suitecontexts']),
                 )
             ));
         }
@@ -1056,6 +1081,20 @@ class behat_config_util {
         return $selectablethemes;
     }
 
+    /**
+     * Return theme directory.
+     *
+     * @param string $themename
+     * @return string theme directory
+     */
+    protected function get_theme_test_directory($themename) {
+        global $CFG;
+
+        $themetestdir = "/theme/" . $themename;
+
+        return $CFG->dirroot . $themetestdir  . self::get_behat_tests_path();
+    }
+
     /**
      * Returns all the directories having overridden tests.
      *
@@ -1070,8 +1109,7 @@ class behat_config_util {
             'contexts' => '|behat_.*\.php$|',
             'features' => '|.*\.feature$|',
         );
-        $themetestdir = "/theme/" . $theme . '/tests/behat';
-        $themetestdirfullpath = $CFG->dirroot . $themetestdir;
+        $themetestdirfullpath = $this->get_theme_test_directory($theme);
 
         // If test directory doesn't exist then return.
         if (!is_dir($themetestdirfullpath)) {
@@ -1081,7 +1119,7 @@ class behat_config_util {
         $directoriestosearch = glob($themetestdirfullpath . DIRECTORY_SEPARATOR . '*' , GLOB_ONLYDIR);
 
         // Include theme directory to find tests.
-        $dirs[realpath($themetestdirfullpath)] = trim(str_replace('/', '_', $themetestdir), '_');
+        $dirs[realpath($themetestdirfullpath)] = trim(str_replace('/', '_', $themetestdirfullpath), '_');
 
         // Search for tests in valid directories.
         foreach ($directoriestosearch as $dir) {
@@ -1108,10 +1146,8 @@ class behat_config_util {
      * @return array list of blacklisted contexts or features
      */
     protected function get_blacklisted_tests_for_theme($theme, $testtype) {
-        global $CFG;
 
-        $themetestpath = $CFG->dirroot . DIRECTORY_SEPARATOR . "theme" . DIRECTORY_SEPARATOR . $theme .
-            self::get_behat_tests_path();
+        $themetestpath = $this->get_theme_test_directory($theme);
 
         if (file_exists($themetestpath . DIRECTORY_SEPARATOR . 'blacklist.json')) {
             // Blacklist file exist. Leave it for last to clear the feature and contexts.
@@ -1168,7 +1204,7 @@ class behat_config_util {
      * Return list of blacklisted behat features for theme and features defined by theme only.
      *
      * @param string $theme theme name.
-     * @return array ($themeblacklistfeatures, $themefeatures)
+     * @return array ($blacklistfeatures, $blacklisttags, $features)
      */
     protected function get_behat_features_for_theme($theme) {
 
@@ -1176,14 +1212,27 @@ class behat_config_util {
         $themefeatures = $this->get_tests_for_theme($theme, 'features');
         $themeblacklistfeatures = $this->get_blacklisted_tests_for_theme($theme, 'features');
 
-        // If tags are specified then we just want features with specified tags.
-        if (!empty($this->tags)) {
-            if (!empty($themefeatures)) {
-                $themefeatures = $this->filtered_features_with_tags($themefeatures);
-            }
+        // Clean feature key and path.
+        $features = array();
+        $blacklistfeatures = array();
+
+        foreach ($themefeatures as $themefeature) {
+            list($featurekey, $featurepath) = $this->get_clean_feature_key_and_path($themefeature);
+            $features[$featurekey] = $featurepath;
+        }
+        foreach ($themeblacklistfeatures as $themeblacklistfeature) {
+            list($blacklistfeaturekey, $blacklistfeaturepath) = $this->get_clean_feature_key_and_path($themeblacklistfeature);
+            $blacklistfeatures[$blacklistfeaturekey] = $blacklistfeaturepath;
         }
 
-        return array($themeblacklistfeatures, $themefeatures);
+        ksort($features);
+
+        $retval = array(
+            'blacklistfeatures' => $blacklistfeatures,
+            'features' => $features
+        );
+
+        return $retval;
     }
 
     /**
@@ -1209,7 +1258,10 @@ class behat_config_util {
 
         // If we already have this list then just return. This will not change by run.
         if (!empty($this->themecontexts[$theme]) && !empty($this->themesuitecontexts)) {
-            return array(array_keys($this->themecontexts[$theme]), $this->themesuitecontexts[$theme]);
+            return array(
+                'contexts' => $this->themecontexts[$theme],
+                'suitecontexts' => $this->themesuitecontexts[$theme],
+            );
         }
 
         if (empty($this->overriddenthemescontexts)) {
@@ -1260,6 +1312,11 @@ class behat_config_util {
         $this->themesuitecontexts[$theme] = $themesuitecontexts;
         $this->themecontexts[$theme] = $themecontexts;
 
-        return array(array_keys($themecontexts), $themesuitecontexts);
+        $retval = array(
+            'contexts' => $themecontexts,
+            'suitecontexts' => $themesuitecontexts,
+        );
+
+        return $retval;
     }
 }
\ No newline at end of file
index c0c0363..11aa866 100644 (file)
@@ -216,12 +216,13 @@ class behat_util extends testing_util {
      * Stores a file in dataroot/behat to allow Moodle to switch
      * to the test environment when using cli-server.
      * @param bool $themesuitewithallfeatures if only theme specific features need to be included in the suite.
+     * @param string $tags comma separated tag, which will be given preference while distributing features in parallel run.
      * @param int $parallelruns number of parallel runs.
      * @param int $run current run.
      * @throws coding_exception
      * @return void
      */
-    public static function start_test_mode($themesuitewithallfeatures = false, $parallelruns = 0, $run = 0) {
+    public static function start_test_mode($themesuitewithallfeatures = false, $tags = '', $parallelruns = 0, $run = 0) {
         global $CFG;
 
         if (!defined('BEHAT_UTIL')) {
@@ -237,7 +238,7 @@ class behat_util extends testing_util {
         self::test_environment_problem();
 
         // Updates all the Moodle features and steps definitions.
-        behat_config_manager::update_config_file('', true, '', $themesuitewithallfeatures, $parallelruns, $run);
+        behat_config_manager::update_config_file('', true, $tags, $themesuitewithallfeatures, $parallelruns, $run);
 
         if (self::is_test_mode_enabled()) {
             return;