Support running multiple random seeds in case one fails

regression-tests/Makefile.simulation-test

@@ -30,8 +30,8 @@ TESTS=$(wildcard ??-*.csc)
 TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
 LOGS=$(patsubst %.csc,%.log,$(TESTS))
 FAILLOGS=$(patsubst %.csc,%.faillog,$(TESTS))
-#Set random seed to create reproduceable results.
-RANDOMSEED=1
+#Set random seeds to create reproduceable results.
+RANDOMSEED=1 5
 
 CONTIKI=../..
 
@@ -57,7 +57,7 @@ RUNALL=false
 endif
 
 %.testlog: %.csc cooja	
-	@$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" "$(RANDOMSEED)"
+	@$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
 
 clean:
 	@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \

regression-tests/simexec.sh

@@ -1,13 +1,23 @@
 #!/bin/bash
+# Do not return an error
 RUNALL=$1
-CSC=$2
-CONTIKI=$3
-BASENAME=$4
-RANDOMSEED=$5
+shift
+# The simulation to run
+CSC=$1
+shift
+#Contiki directory
+CONTIKI=$1
+shift
+#The basename of the experiment
+BASENAME=$1
+shift
+# The test will end on the first successfull run
 
 #set -x
 
-echo -n "Running test $BASENAME "
+while (( "$#" )); do
+	RANDOMSEED=$1
+	echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
 
 	java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
 	JPID=$!
@@ -41,7 +51,7 @@ fi
 
 	# In case of failure
 
-echo " FAIL ಠ_ಠ" | tee -a COOJA.testlog; 
+
 
 	#Verbose output when using CI
 	if [ "$CI" = "true" ];  then 
@@ -51,7 +61,13 @@ else
 		tail -50 COOJA.log ; 
 	fi; 
 
-mv COOJA.testlog $BASENAME.faillog
+	mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
+
+	shift
+done
+
+#All seeds failed
+	echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
 
 # We do not want Make to stop -> Return 0
 if [ "$RUNALL" = "true" ] ; then
@@ -60,7 +76,7 @@ if [ "$RUNALL" = "true" ] ; then
 	exit 0
 fi
 
-#This is a failure
+
 exit 1