Travis tests involving Cooja simulations: when running multiple seeds, fail if any seed fails, rather than succeed if any succeeds.

.travis.yml

@@ -81,7 +81,7 @@ before_script:
 
 script:
   ## tests/Makefile handles most of generic logic
-  - "make -C tests/??-$BUILD_TYPE RUNALL=true summary"
+  - "make -C tests/??-$BUILD_TYPE summary"
 
 after_script:
   ## Print cooja test logs

tests/Makefile

@@ -36,7 +36,7 @@ summary: $(SUMMARIES)
 	grep '' $(SUMMARIES) > summary
 
 summary-%:
-	@make -C $* RUNALL=true summary || true
+	@make -C $* summary || true
 	@echo -n $* | cat - $*/summary > $@
 	@rm $*/summary
 

tests/Makefile.simulation-test

@@ -30,8 +30,9 @@ TESTS=$(wildcard ??-*.csc)
 TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
 LOGS=$(patsubst %.csc,%.log,$(TESTS))
 FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS))
-#Set random seeds to create reproduceable results.
+
-RANDOMSEED=1
+# The number of runs (with different random seeds)
+RUNCOUNT ?= 1
 
 CONTIKI=../..
 
@@ -50,21 +51,13 @@ endif
 
 all: cooja clean tests
 
-ifdef RUNALL
-RUNALL=true
-else
-RUNALL=false
-endif
-
 %.testlog: %.csc cooja
-	@$(CONTIKI)/tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
+	@$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT)
 
 clean:
 	@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
                report summary
 
-
 cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
 $(CONTIKI)/tools/cooja/dist/cooja.jar:
 	(cd $(CONTIKI)/tools/cooja; ant jar)
-

tests/simexec.sh

@@ -1,7 +1,4 @@
 #!/bin/bash
-# Do not return an error
-RUNALL=$1
-shift
 # The simulation to run
 CSC=$1
 shift
@@ -11,15 +8,21 @@ shift
 #The basename of the experiment
 BASENAME=$1
 shift
-# The test will end on the first successfull run
+#The basename of the experiment
+RUNCOUNT=$1
+shift
 
-#set -x
+# Counts all tests run
+declare -i TESTCOUNT=0
 
-while (( "$#" )); do
+# Counts successfull tests
-	RANDOMSEED=$1
+declare -i OKCOUNT=0
-	echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
 
-	java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
+for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do
+	echo -n "Running test $BASENAME with random Seed $SEED"
+
+	# run simulation
+	java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log &
 	JPID=$!
 
 	# Copy the log and only print "." if it changed
@@ -36,23 +39,17 @@ while (( "$#" )); do
 	done
 	rm $BASENAME.log.prog
 
-
+  # wait for end of simulation
 	wait $JPID
 	JRV=$?
 
+  TESTCOUNT+=1
 	if [ $JRV -eq 0 ] ; then
 		touch COOJA.testlog;
 		mv COOJA.testlog $BASENAME.testlog
+		OKCOUNT+=1
 		echo " OK"
-		exit 0
+	else
-	fi
-
-
-
-	# In case of failure
-
-
-
 		# Verbose output when using CI
 		if [ "$CI" = "true" ]; then
 			echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
@@ -64,24 +61,21 @@ while (( "$#" )); do
 			tail -50 $BASENAME.log ;
 		fi;
 
-	mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
+		mv COOJA.testlog $BASENAME.$SEED.faillog
+		echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog;
+	fi
 
 	shift
 done
 
-#All seeds failed
+echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT"
-	echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
 
-# We do not want Make to stop -> Return 0
+if [ $TESTCOUNT -ne $OKCOUNT ] ; then
-if [ "$RUNALL" = "true" ] ; then
+	# At least one test failed
 	touch COOJA.testlog;
 	mv COOJA.testlog $BASENAME.testlog;
-	exit 0
 fi
 
-
+# We do not want Make to stop -> Return 0
-exit 1
+# The Makefile will check if a log contains FAIL at the end
-
+exit 0
-
-
-