diff --git a/.travis.yml b/.travis.yml index d4060c893..a0a39a048 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ before_script: script: ## tests/Makefile handles most of generic logic - - "make -C tests/??-$BUILD_TYPE RUNALL=true summary" + - "make -C tests/??-$BUILD_TYPE summary" after_script: ## Print cooja test logs diff --git a/tests/Makefile b/tests/Makefile index efa9cc54c..8f2e83a1f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -36,7 +36,7 @@ summary: $(SUMMARIES) grep '' $(SUMMARIES) > summary summary-%: - @make -C $* RUNALL=true summary || true + @make -C $* summary || true @echo -n $* | cat - $*/summary > $@ @rm $*/summary diff --git a/tests/Makefile.simulation-test b/tests/Makefile.simulation-test index bbd54703c..cce542a9f 100644 --- a/tests/Makefile.simulation-test +++ b/tests/Makefile.simulation-test @@ -30,8 +30,9 @@ TESTS=$(wildcard ??-*.csc) TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS)) LOGS=$(patsubst %.csc,%.log,$(TESTS)) FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS)) -#Set random seeds to create reproduceable results. -RANDOMSEED=1 + +# The number of runs (with different random seeds) +RUNCOUNT ?= 1 CONTIKI=../.. @@ -50,21 +51,13 @@ endif all: cooja clean tests -ifdef RUNALL -RUNALL=true -else -RUNALL=false -endif - -%.testlog: %.csc cooja - @$(CONTIKI)/tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED) +%.testlog: %.csc cooja + @$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT) clean: @rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \ report summary - cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar $(CONTIKI)/tools/cooja/dist/cooja.jar: (cd $(CONTIKI)/tools/cooja; ant jar) - diff --git a/tests/simexec.sh b/tests/simexec.sh index b4bc91e16..b5535ab34 100755 --- a/tests/simexec.sh +++ b/tests/simexec.sh @@ -1,7 +1,4 @@ #!/bin/bash -# Do not return an error -RUNALL=$1 -shift # The simulation to run CSC=$1 shift @@ -11,15 +8,21 @@ shift #The basename of the experiment BASENAME=$1 shift -# The test will end on the first successfull run +#The basename of the experiment +RUNCOUNT=$1 +shift -#set -x +# Counts all tests run +declare -i TESTCOUNT=0 -while (( "$#" )); do - RANDOMSEED=$1 - echo -n "Running test $BASENAME with random Seed $RANDOMSEED: " +# Counts successfull tests +declare -i OKCOUNT=0 - java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log & +for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do + echo -n "Running test $BASENAME with random Seed $SEED" + + # run simulation + java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log & JPID=$! # Copy the log and only print "." if it changed @@ -28,7 +31,7 @@ while (( "$#" )); do do sleep 1 diff $BASENAME.log $BASENAME.log.prog > /dev/null - if [ $? -ne 0 ] + if [ $? -ne 0 ] then echo -n "." cp $BASENAME.log $BASENAME.log.prog @@ -36,52 +39,43 @@ while (( "$#" )); do done rm $BASENAME.log.prog - + # wait for end of simulation wait $JPID JRV=$? + TESTCOUNT+=1 if [ $JRV -eq 0 ] ; then - touch COOJA.testlog; - mv COOJA.testlog $BASENAME.testlog + touch COOJA.testlog; + mv COOJA.testlog $BASENAME.testlog + OKCOUNT+=1 echo " OK" - exit 0 - fi - - - - # In case of failure - - - - #Verbose output when using CI - if [ "$CI" = "true" ]; then - echo "==== $BASENAME.log ====" ; cat $BASENAME.log; - echo "==== COOJA.testlog ====" ; cat COOJA.testlog; - echo "==== Files used for simulation (sha1sum) ====" - grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum - grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum else - tail -50 $BASENAME.log ; - fi; + # Verbose output when using CI + if [ "$CI" = "true" ]; then + echo "==== $BASENAME.log ====" ; cat $BASENAME.log; + echo "==== COOJA.testlog ====" ; cat COOJA.testlog; + echo "==== Files used for simulation (sha1sum) ====" + grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum + grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum + else + tail -50 $BASENAME.log ; + fi; - mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog + mv COOJA.testlog $BASENAME.$SEED.faillog + echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog; + fi shift done -#All seeds failed - echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog; +echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT" -# We do not want Make to stop -> Return 0 -if [ "$RUNALL" = "true" ] ; then - touch COOJA.testlog; - mv COOJA.testlog $BASENAME.testlog; - exit 0 +if [ $TESTCOUNT -ne $OKCOUNT ] ; then + # At least one test failed + touch COOJA.testlog; + mv COOJA.testlog $BASENAME.testlog; fi - -exit 1 - - - - +# We do not want Make to stop -> Return 0 +# The Makefile will check if a log contains FAIL at the end +exit 0