Add JUnit report generation support to test runner

README.md

@@ -31,6 +31,30 @@ The following options can be set:
 * **`stable `** - Use the latest stable phar release.
 * **`all`** - Use both the latest stable release phar as well as the nightly phar.
 
+### JUnit reports
+
+`$WP_CLI_TEST_JUNIT_DIR` writes JUnit reports to the given directory, in addition to the progress output that is always printed. This is useful to compare the results of two runs, as the reports name every scenario along with the file and line it is defined on, which the progress output does not.
+
+```bash
+WP_CLI_TEST_JUNIT_DIR=build/junit TEST_PACKAGE=wp-cli/entity-command composer test
+```
+
+Behat names each report after the suite it belongs to, so one package results in one report:
+
+```text
+build/junit/wp_cli_entity_command.xml
+build/junit/rerun/wp_cli_entity_command.xml
+```
+
+Failed scenarios are run a second time. Reports of that rerun are written to the `rerun` subdirectory, as they only cover the scenarios that failed the first time and would otherwise overwrite the full report of the package.
+
+The rerun report holds a result for every scenario it ran, which is what tells a flaky scenario from a failing one:
+
+* A scenario reported as `failed` in the report of the package and as `passed` in the rerun report failed once and then passed, and was therefore flaky.
+* A scenario reported as `failed` in both is failing consistently.
+
+Reports left behind by an earlier run are removed before the tests start, so that the directory only ever describes the run that wrote it. A package that passes produces no rerun report at all, which would otherwise leave a stale one in place.
+
 ### Automated Builds
 
 This repository is being rebuilt through a Travis CI cron job every 24 hours to post test results in Emails and Slack.

bin/test-source

@@ -9,6 +9,16 @@
 # - "all": The framework as well as all bundled commands are tested.
 # - "commands": Only the command packages are tested.
 # - <package name>: Only the package named <package name> is tested.
+#
+# $WP_CLI_TEST_JUNIT_DIR optionally writes JUnit reports to the given directory,
+# in addition to the progress output. Behat names each report after the suite it
+# belongs to, so one package results in one report.
+#
+# Reports of the rerun of the failed scenarios are written to a "rerun"
+# subdirectory, as they only cover the scenarios that failed the first time and
+# would otherwise overwrite the full report of the package. The rerun report
+# holds a result for every scenario it ran: one reported as passed there was
+# flaky, one reported as failed is failing consistently.
 
 set -e
 
@@ -39,6 +49,24 @@ if [ "$TEST_PACKAGE" != "all" -a "$TEST_PACKAGE" != "commands" ]; then
 	REPOS="$TEST_PACKAGE"
 fi
 
+JUNIT_ARGS=()
+JUNIT_RERUN_ARGS=()
+
+if [ -n "$WP_CLI_TEST_JUNIT_DIR" ]; then
+	# A package that passes does not produce a rerun report at all, so a report
+	# left behind by an earlier run would be read as belonging to this one.
+	rm -f "${WP_CLI_TEST_JUNIT_DIR}"/*.xml "${WP_CLI_TEST_JUNIT_DIR}"/rerun/*.xml
+
+	mkdir -p "${WP_CLI_TEST_JUNIT_DIR}/rerun"
+
+	# The first --out belongs to the progress format and keeps it on stdout, the
+	# second one is the directory the JUnit reports are written to.
+	JUNIT_ARGS=(--format junit --out std --out "${WP_CLI_TEST_JUNIT_DIR}")
+	JUNIT_RERUN_ARGS=(--format junit --out std --out "${WP_CLI_TEST_JUNIT_DIR}/rerun")
+
+	echo "Writing JUnit reports to ${WP_CLI_TEST_JUNIT_DIR}"
+fi
+
 for REPO in $REPOS; do
 
 	echo "Testing package $REPO..."
@@ -47,9 +75,9 @@ for REPO in $REPOS; do
 	echo "Behat Tags: $BEHAT_TAGS"
 
 	set +e
-	"${BUILD_DIR}/vendor/bin/behat" --format progress $BEHAT_TAGS --strict  --suite $REPO
+	"${BUILD_DIR}/vendor/bin/behat" --format progress "${JUNIT_ARGS[@]}" $BEHAT_TAGS --strict  --suite $REPO
 	if [ $? -ne 0 ]; then
-		"${BUILD_DIR}/vendor/bin/behat" --format progress $BEHAT_TAGS --strict  --suite $REPO --rerun
+		"${BUILD_DIR}/vendor/bin/behat" --format progress "${JUNIT_RERUN_ARGS[@]}" $BEHAT_TAGS --strict  --suite $REPO --rerun
 		if [ $? -ne 0 ]; then
 			FAILED_PACKAGES="$FAILED_PACKAGES ${RELEASE}:${REPO}"
 		fi