# RVL generic regression tester
#
# WWS, 9/09
#
# general: to be used with rest of TRIP build system, including
# maw, in usual project directory structure. Regression tests are
# best confined to directories reserved for the purpose.
#
# directory setup
# * all executables must be unit tests
# * Orig directory must be provided, home for reference results
 
# a unit test must 
# * be a self-contained executable, requiring no input args.
# * trap exceptions - place driver code in try block, write 
#     exception and exit(1) if exception is trapped, else return 0
# * output text info to stdout which can be tested against ref
#     output to gauge correct execution. may produce other output
#     (binary or ascii) but only the output written to stdout will
#     be used for verification.
# * the text output should be INDEPENDENT OF PLATFORM, COMPILE OPTIONS,
#   AND RUNTIME ENVIRONMENT. 
#
# unit tests should be "small", in sense that total runtime for all
# tests in directory is "acceptable"
# 
# targets
#
# main target is regress, which produces a regression report
# regress.rpt describing the verification results. Other targets:
#
# * regress.install: makes any absent reference results, store in Orig - use
#     this target for initial setup of reference files. NOTE: BECAUSE ORIG 
#     DOES NOT CLEAN, ALL USES AFTER FIRST ARE NO-OPS.
# * regress_<name>: makes regression output for executable <name>.x
# * regress_<name>.install: makes reference output for <name>.x
# * rclean: remove all files related to regression
# * clean: local clean rule, includes rclean
#
# Because reference files are stored in Orig, in which maw inserts no 
# makefile, they are not removed by any version of clean or burn. Therefor,
# revising reference output requires removing the reference file(s) in Orig
# BY HAND.

# for executation in environments other than the unix command line, use 
# the EXEC macro - override it in the command line or in another makefile 
# fragment. for example, mpi execution might be enabled by
#
# EXEC=mpiexec ...(parameters) 

EXEC=

regress_%.ref: %.x
	@(if [ ! -f Orig/$@ ] ; then $(EXEC) ./$< > Orig/$@ ; fi)
	@(ln -s Orig/$@ .)

regress.install: ${BINS}
	@(for i in $(BINS:.x=.ref) ; do \
	  $(MAKE) regress_$$i ; \
	done) 

regress_%: %.x
	@($(MAKE) $@.ref)
	@($(EXEC) ./$< > $@; if [ $$? != 0 ] ; then echo "$< abort" ; else ( if [ -z "`diff -q $@ $@.ref`" ] ; then echo "$< normal termination; output identical to reference" ; else echo "$< normal termination; output differs from reference" ; fi) ; fi)

regress: ${BINS}
	@($(MAKE) rclean ; \
	echo " "                             > regress.rpt ; \
	echo "****************************" >> regress.rpt ; \
	echo "* RVLTOOLS REGRESSION TEST *" >> regress.rpt ; \
	echo "****************************" >> regress.rpt ; \
	echo "package = $(PACKAGE)"         >> regress.rpt ; \
	echo " "                            >> regress.rpt ; \
	for i in $(BINS:.x=) ; do \
	  $(MAKE) -i regress_$$i | grep -v make >> regress.rpt ; \
	done )

rclean:
	@(rm -f regress_* *.rpt)

clean: jclean rclean