giomba/nes-proj
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Travis tests involving Cooja simulations: when running multiple seeds, fail if any seed fails, rather than succeed if any succeeds.

Browse Source
This commit is contained in:
Simon Duquennoy 2017-11-05 14:57:31 +01:00 committed by Simon Duquennoy
parent a7251460a4
commit ac3c7fb51c
4 changed files with 46 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -81,7 +81,7 @@ before_script:
script:
## tests/Makefile handles most of generic logic
- "make -C tests/??-$BUILD_TYPE RUNALL=true summary"
- "make -C tests/??-$BUILD_TYPE summary"
after_script:
## Print cooja test logs

2
tests/Makefile
View File

@ -36,7 +36,7 @@ summary: $(SUMMARIES)
grep '' $(SUMMARIES) > summary
summary-%:
@make -C $* RUNALL=true summary || true
@make -C $* summary || true
@echo -n $* | cat - $*/summary > $@
@rm $*/summary

15
tests/Makefile.simulation-test
View File

@ -30,8 +30,9 @@ TESTS=$(wildcard ??-*.csc)
TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
LOGS=$(patsubst %.csc,%.log,$(TESTS))
FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS))
#Set random seeds to create reproduceable results.
RANDOMSEED=1
# The number of runs (with different random seeds)
RUNCOUNT ?= 1
CONTIKI=../..
@ -50,21 +51,13 @@ endif
all: cooja clean tests
ifdef RUNALL
RUNALL=true
else
RUNALL=false
endif
%.testlog: %.csc cooja
@$(CONTIKI)/tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
@$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT)
clean:
@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
report summary
cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
$(CONTIKI)/tools/cooja/dist/cooja.jar:
(cd $(CONTIKI)/tools/cooja; ant jar)

74
tests/simexec.sh
View File

@ -1,7 +1,4 @@
#!/bin/bash
# Do not return an error
RUNALL=$1
shift
# The simulation to run
CSC=$1
shift
@ -11,15 +8,21 @@ shift
#The basename of the experiment
BASENAME=$1
shift
# The test will end on the first successfull run
#The basename of the experiment
RUNCOUNT=$1
shift
#set -x
# Counts all tests run
declare -i TESTCOUNT=0
while (( "$#" )); do
RANDOMSEED=$1
echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
# Counts successfull tests
declare -i OKCOUNT=0
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do
echo -n "Running test $BASENAME with random Seed $SEED"
# run simulation
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log &
JPID=$!
# Copy the log and only print "." if it changed
@ -36,52 +39,43 @@ while (( "$#" )); do
done
rm $BASENAME.log.prog
# wait for end of simulation
wait $JPID
JRV=$?
TESTCOUNT+=1
if [ $JRV -eq 0 ] ; then
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog
OKCOUNT+=1
echo " OK"
exit 0
fi
# In case of failure
#Verbose output when using CI
if [ "$CI" = "true" ]; then
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
echo "==== Files used for simulation (sha1sum) ===="
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
else
tail -50 $BASENAME.log ;
fi;
# Verbose output when using CI
if [ "$CI" = "true" ]; then
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
echo "==== Files used for simulation (sha1sum) ===="
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
else
tail -50 $BASENAME.log ;
fi;
mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
mv COOJA.testlog $BASENAME.$SEED.faillog
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog;
fi
shift
done
#All seeds failed
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT"
# We do not want Make to stop -> Return 0
if [ "$RUNALL" = "true" ] ; then
if [ $TESTCOUNT -ne $OKCOUNT ] ; then
# At least one test failed
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog;
exit 0
fi
exit 1
# We do not want Make to stop -> Return 0
# The Makefile will check if a log contains FAIL at the end
exit 0