Travis tests involving Cooja simulations: when running multiple seeds, fail if any seed fails, rather than succeed if any succeeds.
This commit is contained in:
Simon Duquennoy
Simon Duquennoy
parent
a7251460a4
commit
ac3c7fb51c
@ -81,7 +81,7 @@ before_script:
|
||||
|
||||
script:
|
||||
## tests/Makefile handles most of generic logic
|
||||
- "make -C tests/??-$BUILD_TYPE RUNALL=true summary"
|
||||
- "make -C tests/??-$BUILD_TYPE summary"
|
||||
|
||||
after_script:
|
||||
## Print cooja test logs
|
||||
|
||||
@ -36,7 +36,7 @@ summary: $(SUMMARIES)
|
||||
grep '' $(SUMMARIES) > summary
|
||||
|
||||
summary-%:
|
||||
@make -C $* RUNALL=true summary || true
|
||||
@make -C $* summary || true
|
||||
@echo -n $* | cat - $*/summary > $@
|
||||
@rm $*/summary
|
||||
|
||||
|
||||
@ -30,8 +30,9 @@ TESTS=$(wildcard ??-*.csc)
|
||||
TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
|
||||
LOGS=$(patsubst %.csc,%.log,$(TESTS))
|
||||
FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS))
|
||||
#Set random seeds to create reproduceable results.
|
||||
RANDOMSEED=1
|
||||
|
||||
# The number of runs (with different random seeds)
|
||||
RUNCOUNT ?= 1
|
||||
|
||||
CONTIKI=../..
|
||||
|
||||
@ -50,21 +51,13 @@ endif
|
||||
|
||||
all: cooja clean tests
|
||||
|
||||
ifdef RUNALL
|
||||
RUNALL=true
|
||||
else
|
||||
RUNALL=false
|
||||
endif
|
||||
|
||||
%.testlog: %.csc cooja
|
||||
@$(CONTIKI)/tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
|
||||
%.testlog: %.csc cooja
|
||||
@$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT)
|
||||
|
||||
clean:
|
||||
@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
|
||||
report summary
|
||||
|
||||
|
||||
cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
|
||||
$(CONTIKI)/tools/cooja/dist/cooja.jar:
|
||||
(cd $(CONTIKI)/tools/cooja; ant jar)
|
||||
|
||||
|
||||
@ -1,7 +1,4 @@
|
||||
#!/bin/bash
|
||||
# Do not return an error
|
||||
RUNALL=$1
|
||||
shift
|
||||
# The simulation to run
|
||||
CSC=$1
|
||||
shift
|
||||
@ -11,15 +8,21 @@ shift
|
||||
#The basename of the experiment
|
||||
BASENAME=$1
|
||||
shift
|
||||
# The test will end on the first successfull run
|
||||
#The basename of the experiment
|
||||
RUNCOUNT=$1
|
||||
shift
|
||||
|
||||
#set -x
|
||||
# Counts all tests run
|
||||
declare -i TESTCOUNT=0
|
||||
|
||||
while (( "$#" )); do
|
||||
RANDOMSEED=$1
|
||||
echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
|
||||
# Counts successfull tests
|
||||
declare -i OKCOUNT=0
|
||||
|
||||
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
|
||||
for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do
|
||||
echo -n "Running test $BASENAME with random Seed $SEED"
|
||||
|
||||
# run simulation
|
||||
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log &
|
||||
JPID=$!
|
||||
|
||||
# Copy the log and only print "." if it changed
|
||||
@ -28,7 +31,7 @@ while (( "$#" )); do
|
||||
do
|
||||
sleep 1
|
||||
diff $BASENAME.log $BASENAME.log.prog > /dev/null
|
||||
if [ $? -ne 0 ]
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo -n "."
|
||||
cp $BASENAME.log $BASENAME.log.prog
|
||||
@ -36,52 +39,43 @@ while (( "$#" )); do
|
||||
done
|
||||
rm $BASENAME.log.prog
|
||||
|
||||
|
||||
# wait for end of simulation
|
||||
wait $JPID
|
||||
JRV=$?
|
||||
|
||||
TESTCOUNT+=1
|
||||
if [ $JRV -eq 0 ] ; then
|
||||
touch COOJA.testlog;
|
||||
mv COOJA.testlog $BASENAME.testlog
|
||||
touch COOJA.testlog;
|
||||
mv COOJA.testlog $BASENAME.testlog
|
||||
OKCOUNT+=1
|
||||
echo " OK"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
|
||||
# In case of failure
|
||||
|
||||
|
||||
|
||||
#Verbose output when using CI
|
||||
if [ "$CI" = "true" ]; then
|
||||
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
|
||||
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
|
||||
echo "==== Files used for simulation (sha1sum) ===="
|
||||
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
|
||||
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
|
||||
else
|
||||
tail -50 $BASENAME.log ;
|
||||
fi;
|
||||
# Verbose output when using CI
|
||||
if [ "$CI" = "true" ]; then
|
||||
echo "==== $BASENAME.log ====" ; cat $BASENAME.log;
|
||||
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
|
||||
echo "==== Files used for simulation (sha1sum) ===="
|
||||
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
|
||||
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
|
||||
else
|
||||
tail -50 $BASENAME.log ;
|
||||
fi;
|
||||
|
||||
mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
|
||||
mv COOJA.testlog $BASENAME.$SEED.faillog
|
||||
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog;
|
||||
fi
|
||||
|
||||
shift
|
||||
done
|
||||
|
||||
#All seeds failed
|
||||
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
|
||||
echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT"
|
||||
|
||||
# We do not want Make to stop -> Return 0
|
||||
if [ "$RUNALL" = "true" ] ; then
|
||||
touch COOJA.testlog;
|
||||
mv COOJA.testlog $BASENAME.testlog;
|
||||
exit 0
|
||||
if [ $TESTCOUNT -ne $OKCOUNT ] ; then
|
||||
# At least one test failed
|
||||
touch COOJA.testlog;
|
||||
mv COOJA.testlog $BASENAME.testlog;
|
||||
fi
|
||||
|
||||
|
||||
exit 1
|
||||
|
||||
|
||||
|
||||
|
||||
# We do not want Make to stop -> Return 0
|
||||
# The Makefile will check if a log contains FAIL at the end
|
||||
exit 0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user