giomba/nes-proj
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Travis tests involving Cooja simulations: when running multiple seeds, fail if any seed fails, rather than succeed if any succeeds.

Browse Source
This commit is contained in:
Simon Duquennoy 2017-11-05 14:57:31 +01:00 committed by Simon Duquennoy
parent a7251460a4
commit ac3c7fb51c
4 changed files with 46 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -81,7 +81,7 @@ before_script:
script: script:
## tests/Makefile handles most of generic logic ## tests/Makefile handles most of generic logic
- "make -C tests/??-$BUILD_TYPE RUNALL=true summary" - "make -C tests/??-$BUILD_TYPE summary"
after_script: after_script:
## Print cooja test logs ## Print cooja test logs

2
tests/Makefile
View File

@ -36,7 +36,7 @@ summary: $(SUMMARIES)
grep '' $(SUMMARIES) > summary grep '' $(SUMMARIES) > summary
summary-%: summary-%:
@make -C $* RUNALL=true summary || true @make -C $* summary || true
@echo -n $* | cat - $*/summary > $@ @echo -n $* | cat - $*/summary > $@
@rm $*/summary @rm $*/summary

17
tests/Makefile.simulation-test
View File

@ -30,8 +30,9 @@ TESTS=$(wildcard ??-*.csc)
TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS)) TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
LOGS=$(patsubst %.csc,%.log,$(TESTS)) LOGS=$(patsubst %.csc,%.log,$(TESTS))
FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS)) FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS))
#Set random seeds to create reproduceable results.
RANDOMSEED=1 # The number of runs (with different random seeds)
RUNCOUNT ?= 1
CONTIKI=../.. CONTIKI=../..
@ -50,21 +51,13 @@ endif
all: cooja clean tests all: cooja clean tests
ifdef RUNALL %.testlog: %.csc cooja
RUNALL=true @$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT)
else
RUNALL=false
endif
%.testlog: %.csc cooja
@$(CONTIKI)/tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
clean: clean:
@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \ @rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
report summary report summary
cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
$(CONTIKI)/tools/cooja/dist/cooja.jar: $(CONTIKI)/tools/cooja/dist/cooja.jar:
(cd $(CONTIKI)/tools/cooja; ant jar) (cd $(CONTIKI)/tools/cooja; ant jar)

84
tests/simexec.sh
View File

@ -1,7 +1,4 @@
#!/bin/bash #!/bin/bash
# Do not return an error
RUNALL=$1
shift
# The simulation to run # The simulation to run
CSC=$1 CSC=$1
shift shift
@ -11,15 +8,21 @@ shift
#The basename of the experiment #The basename of the experiment
BASENAME=$1 BASENAME=$1
shift shift
# The test will end on the first successfull run #The basename of the experiment
RUNCOUNT=$1
shift
#set -x # Counts all tests run
declare -i TESTCOUNT=0
while (( "$#" )); do # Counts successfull tests
RANDOMSEED=$1 declare -i OKCOUNT=0
echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log & for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do
echo -n "Running test $BASENAME with random Seed $SEED"
# run simulation
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log &
JPID=$! JPID=$!
# Copy the log and only print "." if it changed # Copy the log and only print "." if it changed
@ -28,7 +31,7 @@ while (( "$#" )); do
do do
sleep 1 sleep 1
diff $BASENAME.log $BASENAME.log.prog > /dev/null diff $BASENAME.log $BASENAME.log.prog > /dev/null
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then
echo -n "." echo -n "."
cp $BASENAME.log $BASENAME.log.prog cp $BASENAME.log $BASENAME.log.prog
@ -36,52 +39,43 @@ while (( "$#" )); do
done done
rm $BASENAME.log.prog rm $BASENAME.log.prog
# wait for end of simulation
wait $JPID wait $JPID
JRV=$? JRV=$?
TESTCOUNT+=1
if [ $JRV -eq 0 ] ; then if [ $JRV -eq 0 ] ; then
touch COOJA.testlog; touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog mv COOJA.testlog $BASENAME.testlog
OKCOUNT+=1
echo " OK" echo " OK"
exit 0
fi
# In case of failure
#Verbose output when using CI
if [ "$CI" = "true" ]; then
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
echo "==== Files used for simulation (sha1sum) ===="
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
else else
tail -50 $BASENAME.log ; # Verbose output when using CI
fi; if [ "$CI" = "true" ]; then
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
echo "==== Files used for simulation (sha1sum) ===="
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
else
tail -50 $BASENAME.log ;
fi;
mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog mv COOJA.testlog $BASENAME.$SEED.faillog
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog;
fi
shift shift
done done
#All seeds failed echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT"
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
# We do not want Make to stop -> Return 0 if [ $TESTCOUNT -ne $OKCOUNT ] ; then
if [ "$RUNALL" = "true" ] ; then # At least one test failed
touch COOJA.testlog; touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog; mv COOJA.testlog $BASENAME.testlog;
exit 0
fi fi
# We do not want Make to stop -> Return 0
exit 1 # The Makefile will check if a log contains FAIL at the end
exit 0