fate: Add --ignore-tests configure option for omitting specific FATE tests

Message ID 20161227221949.46072-1-martin@martin.st
State Committed
Headers show

Commit Message

Martin Storsjö Dec. 27, 2016, 10:19 p.m.
From: Janne Grunau <janne-libav@jannau.net>

This can be useful to filter out noise in known-broken scenarios like
miscompilation by legacy compilers and similar.

Originally based on a patch by Diego Biurrun.
---
This is the common agreed upon consensus from Diego's and Janne's
discussion, as far as I read it.
---
 configure         | 11 +++++++++++
 doc/fate.texi     |  1 +
 tests/Makefile    |  4 +++-
 tests/fate-run.sh |  9 +++++++--
 tests/fate.sh     |  1 +
 5 files changed, 23 insertions(+), 3 deletions(-)

Comments

Diego Biurrun Dec. 28, 2016, 4:38 p.m. | #1
On Wed, Dec 28, 2016 at 12:19:49AM +0200, Martin Storsjö wrote:
> --- a/configure
> +++ b/configure
> @@ -348,6 +348,8 @@ Developer options (useful when working on Libav itself):
>    --random-seed=VALUE      seed value for --enable/disable-random
>    --disable-valgrind-backtrace do not print a backtrace under Valgrind
>                             (only applies to --disable-optimizations builds)
> +  --ignore-tests=TESTS     comma-separated list (without "fate-" prefix
> +                           in the name) of tests which result is ignored

whose result

Thanks for fixing this up, LGTM with the grammar fixed.

Diego
Diego Biurrun Dec. 28, 2016, 5:16 p.m. | #2
On Wed, Dec 28, 2016 at 05:38:17PM +0100, Diego Biurrun wrote:
> On Wed, Dec 28, 2016 at 12:19:49AM +0200, Martin Storsjö wrote:
> > --- a/configure
> > +++ b/configure
> > @@ -348,6 +348,8 @@ Developer options (useful when working on Libav itself):
> >    --random-seed=VALUE      seed value for --enable/disable-random
> >    --disable-valgrind-backtrace do not print a backtrace under Valgrind
> >                             (only applies to --disable-optimizations builds)
> > +  --ignore-tests=TESTS     comma-separated list (without "fate-" prefix
> > +                           in the name) of tests which result is ignored
> 
> whose result
> 
> Thanks for fixing this up, LGTM with the grammar fixed.

Pushed with that changed and with the output changed from

  ignoring foo

to

  IGNORE  foo

so that it matches the look of other output from the build system.

Diego

Patch

diff --git a/configure b/configure
index 8e40238..8472145 100755
--- a/configure
+++ b/configure
@@ -348,6 +348,8 @@  Developer options (useful when working on Libav itself):
   --random-seed=VALUE      seed value for --enable/disable-random
   --disable-valgrind-backtrace do not print a backtrace under Valgrind
                            (only applies to --disable-optimizations builds)
+  --ignore-tests=TESTS     comma-separated list (without "fate-" prefix
+                           in the name) of tests which result is ignored
 
 NOTE: Object files are built at the place where configure is launched.
 EOF
@@ -1815,6 +1817,7 @@  CMDLINE_SET="
     host_ld
     host_ldflags
     host_os
+    ignore_tests
     ld
     logfile
     malloc_prefix
@@ -5192,6 +5195,13 @@  for type in decoder encoder hwaccel parser demuxer muxer protocol filter bsf ind
     echo
 done
 
+if test -n "$ignore_tests"; then
+    ignore_tests=$(echo $ignore_tests | tr ',' ' ')
+    echo "Ignored FATE tests:"
+    echo $ignore_tests | print_3_columns
+    echo
+fi
+
 license="LGPL version 2.1 or later"
 if enabled nonfree; then
     license="nonfree and unredistributable"
@@ -5316,6 +5326,7 @@  SLIB_INSTALL_EXTRA_LIB=${SLIB_INSTALL_EXTRA_LIB}
 SLIB_INSTALL_EXTRA_SHLIB=${SLIB_INSTALL_EXTRA_SHLIB}
 VERSION_SCRIPT_POSTPROCESS_CMD=${VERSION_SCRIPT_POSTPROCESS_CMD}
 SAMPLES:=${samples:-\$(LIBAV_SAMPLES)}
+IGNORE_TESTS=$ignore_tests
 EOF
 
 map 'eval echo "${v}_FFLIBS=\$${v}_deps" >> avbuild/config.mak' $LIBRARY_LIST
diff --git a/doc/fate.texi b/doc/fate.texi
index 9e654e7..b1bfa2e 100644
--- a/doc/fate.texi
+++ b/doc/fate.texi
@@ -139,6 +139,7 @@  workdir=                                # directory in which to do all the work
 fate_recv="ssh -T fate@@fate.libav.org"  # command to submit report
 comment=                                # optional description
 build_only=     # set to "yes" for a compile-only instance that skips tests
+ignore_tests=
 
 # the following are optional and map to configure options
 arch=
diff --git a/tests/Makefile b/tests/Makefile
index 0e475a2..30e06e8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -143,11 +143,13 @@  endif
 
 FATE_UTILS = base64 tiny_psnr
 
+$(addprefix fate-, $(IGNORE_TESTS)): REPORT=ignore
+
 fate: $(FATE)
 
 $(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF))
 	@echo "TEST    $(@:fate-%=%)"
-	$(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)'
+	$(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)'
 
 fate-list:
 	@printf '%s\n' $(sort $(FATE))
diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index b1b299a..3982a18 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -24,6 +24,7 @@  size_tolerance=${14:-0}
 cmp_unit=${15:-2}
 gen=${16:-no}
 hwaccel=${17:-none}
+report_type=${18:-standard}
 
 outdir="tests/data/fate"
 outfile="${outdir}/${test}"
@@ -212,13 +213,17 @@  if test -e "$ref" || test $cmp = "oneline" ; then
     esac
     cmperr=$?
     test $err = 0 && err=$cmperr
-    test $err = 0 || cat $cmpfile
+    if [ "$report_type" = "ignore" ]; then
+        test $err = 0 || echo "ignoring fate-${test}" && err=0
+    else
+        test $err = 0 || cat $cmpfile
+    fi
 else
     echo "reference file '$ref' not found"
     err=1
 fi
 
-if [ $err -eq 0 ]; then
+if [ $err -eq 0 ] && test $report_type = "standard" ; then
     unset cmpo erro
 else
     cmpo="$($base64 <$cmpfile)"
diff --git a/tests/fate.sh b/tests/fate.sh
index 4608d2d..c93e20a 100755
--- a/tests/fate.sh
+++ b/tests/fate.sh
@@ -47,6 +47,7 @@  configure()(
         --prefix="${inst}"                                              \
         --samples="${samples}"                                          \
         --enable-gpl                                                    \
+        ${ignore_tests:+--ignore-tests="$ignore_tests"}                 \
         ${arch:+--arch=$arch}                                           \
         ${cpu:+--cpu="$cpu"}                                            \
         ${toolchain:+--toolchain="$toolchain"}                          \