[i-g-t,3/4] intel-ci: Build-time check for testlists

Commit Message

Petri Latvala March 17, 2017, 1:25 p.m. UTC

Test at check and distcheck that testlist files only contain tests
that are still present.

Signed-off-by: Petri Latvala <[e-mail]>
---
 configure.ac                      |  1 +
 tests/Makefile.am                 |  2 ++
 tests/intel-ci/.gitignore         |  1 +
 tests/intel-ci/Makefile.am        | 42 +++++++++++++++++++++++++
 tests/intel-ci/check_testlists.sh | 65 +++++++++++++++++++++++++++++++++++++++
 tests/intel-ci/checked_lists.txt  |  1 +
 6 files changed, 112 insertions(+)
 create mode 100644 tests/intel-ci/.gitignore
 create mode 100644 tests/intel-ci/Makefile.am
 create mode 100755 tests/intel-ci/check_testlists.sh
 create mode 100644 tests/intel-ci/checked_lists.txt

Patch

diff --git a/configure.ac b/configure.ac
index 12b0ab0..9aa6068 100644
--- a/configure.ac
+++ b/configure.ac
@@ -341,6 +341,7 @@  AC_CONFIG_FILES([
 		 man/Makefile
 		 scripts/Makefile
 		 tests/Makefile
+		 tests/intel-ci/Makefile
 		 tools/Makefile
 		 tools/null_state_gen/Makefile
 		 tools/registers/Makefile
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2080b3f..001106f 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,5 +1,7 @@ 
 include Makefile.sources
 
+SUBDIRS = . intel-ci
+
 if HAVE_LIBDRM_NOUVEAU
     TESTS_progs_M += $(NOUVEAU_TESTS_M)
 endif
diff --git a/tests/intel-ci/.gitignore b/tests/intel-ci/.gitignore
new file mode 100644
index 0000000..1a8f133
--- /dev/null
+++ b/tests/intel-ci/.gitignore
@@ -0,0 +1 @@ 
+all-tests.testlist
diff --git a/tests/intel-ci/Makefile.am b/tests/intel-ci/Makefile.am
new file mode 100644
index 0000000..de92b24
--- /dev/null
+++ b/tests/intel-ci/Makefile.am
@@ -0,0 +1,42 @@ 
+if BUILD_TESTS
+
+TESTLIST = $(top_builddir)/tests/test-list.txt
+
+ALL_TESTS = all-tests.testlist
+CHECKED_TESTLISTS = checked_lists.txt
+
+check_DATA = $(ALL_TESTS)
+$(ALL_TESTS): $(TESTLIST)
+	echo "# This file is automatically generated at build-time." > $@
+	echo "# It contains a list of all available (sub)tests, " >> $@
+	echo "# for checking other lists against it." >> $@
+	echo >> $@
+	for test in `cat $(TESTLIST)`; do \
+		if [ "$$test" = "TESTLIST" -o "$$test" = "END" ]; then \
+			continue; \
+		fi; \
+		if [ -x $(top_builddir)/tests/$$test ]; then \
+			testprog=$(top_builddir)/tests/$$test; \
+		else \
+			testprog=$(top_srcdir)/tests/$$test; \
+		fi; \
+		progbase=`basename $$testprog | tr '[:upper:]' '[:lower:]'`; \
+		if ./$$testprog --list-subtests > /dev/null ; then \
+			for subtest in `./$$testprog --list-subtests`; do \
+				subname=`echo $$subtest | tr '[:upper:]' '[:lower:]'`; \
+				echo "igt@$$progbase@$$subname" >> $@; \
+			done; \
+		else \
+			echo "igt@$$progbase" >> $@; \
+		fi; \
+	done
+
+CLEANFILES = $(ALL_TESTS)
+
+check_SCRIPTS = check_testlists.sh
+
+EXTRA_DIST = $(CHECKED_TESTLISTS) $(shell cat $(CHECKED_TESTLISTS)) $(check_SCRIPTS)
+
+TESTS = $(check_SCRIPTS)
+
+endif
diff --git a/tests/intel-ci/check_testlists.sh b/tests/intel-ci/check_testlists.sh
new file mode 100755
index 0000000..648364d
--- /dev/null
+++ b/tests/intel-ci/check_testlists.sh
@@ -0,0 +1,65 @@ 
+#!/bin/sh
+#
+# Copyright © 2017 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+#
+# Check that testlist files don't have tests that are not available
+#
+
+if [ -z "$top_builddir" ]; then
+	top_builddir="$(dirname $0)"
+fi
+
+# allow to run this script from top directory
+TESTLISTS=`cat $top_builddir/checked_lists.txt`
+if [ $? -ne 0 ]; then
+	# distcheck requires this hack
+	TESTLISTS=$(cat checked_lists.txt)
+	if [ $? -ne 0 ]; then
+		echo "Error: Could not read list of testlists"
+		exit 99
+	fi
+fi
+
+ALLTESTS="$top_builddir/all-tests.testlist"
+if [ ! -r "$ALLTESTS" ]; then
+	# distcheck requires this hack
+	ALLTESTS=all-tests.testlist
+	if [ ! -r "$ALLTESTS" ]; then
+		echo "Error: Could not read list of all tests"
+		exit 99
+	fi
+fi
+
+failure=0
+fail () {
+	echo "FAIL: Test $1 in test list $2 does not exist"
+	failure=1
+}
+
+for testlist in $TESTLISTS; do
+	for testname in `cat "$testlist" | sed 's/#.*$//g'`; do
+		grep -q "^$testname\$" "$ALLTESTS" || fail "$testname" "$testlist"
+	done
+done
+
+exit $failure
diff --git a/tests/intel-ci/checked_lists.txt b/tests/intel-ci/checked_lists.txt
new file mode 100644
index 0000000..2f8a386
--- /dev/null
+++ b/tests/intel-ci/checked_lists.txt
@@ -0,0 +1 @@ 
+fast-feedback.testlist