giomba/nes-proj
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

CI simulation tests: cleaner output and simplified test logic

Browse Source
This commit is contained in:
Simon Duquennoy 2017-11-11 13:32:48 +01:00 committed by Simon Duquennoy
parent ac3c7fb51c
commit 1487f60289
2 changed files with 33 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tests/Makefile.simulation-test
View File

@ -28,9 +28,9 @@
TESTS=$(wildcard ??-*.csc) TESTS=$(wildcard ??-*.csc)
TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS)) TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
LOGS=$(patsubst %.csc,%.log,$(TESTS))
FAILLOGS=$(patsubst %.csc,%.*.faillog,$(TESTS))
# The random seed to start from
BASESEED ?= 1
# The number of runs (with different random seeds) # The number of runs (with different random seeds)
RUNCOUNT ?= 1 RUNCOUNT ?= 1
@ -38,25 +38,20 @@ CONTIKI=../..
tests: $(TESTLOGS) tests: $(TESTLOGS)
report: clean tests summary: clean tests
@echo | grep -s -e '' - $(LOGS) $(TESTLOGS) $(FAILLOGS) > $@ || true
summary: report
ifeq ($(TESTS),) ifeq ($(TESTS),)
@echo No tests > $@ @echo No tests > $@
else else
@egrep -e ' OK| FAIL' $< > $@ @cat $(TESTLOGS) > $@
@ls -1 ??-*.faillog > /dev/null 2>&1; [ $$? = 0 ] && tail -v ??-*.log ??-*.faillog >> $@ || true
endif endif
all: cooja clean tests all: cooja clean tests
%.testlog: %.csc cooja %.testlog: %.csc cooja
@$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(RUNCOUNT) @$(CONTIKI)/tests/simexec.sh "$<" "$(CONTIKI)" "$(basename $@)" $(BASESEED) $(RUNCOUNT)
clean: clean:
@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \ @rm -f *.*log report summary
report summary
cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar cooja: $(CONTIKI)/tools/cooja/dist/cooja.jar
$(CONTIKI)/tools/cooja/dist/cooja.jar: $(CONTIKI)/tools/cooja/dist/cooja.jar:

52
tests/simexec.sh
View File

@ -8,7 +8,10 @@ shift
#The basename of the experiment #The basename of the experiment
BASENAME=$1 BASENAME=$1
shift shift
#The basename of the experiment #The random seed to start from
BASESEED=$1
shift
#The number of runs (with different seeds)
RUNCOUNT=$1 RUNCOUNT=$1
shift shift
@ -18,62 +21,61 @@ declare -i TESTCOUNT=0
# Counts successfull tests # Counts successfull tests
declare -i OKCOUNT=0 declare -i OKCOUNT=0
for (( SEED=1; SEED<=$RUNCOUNT; SEED++ )); do # A list of seeds the resulted in failure
FAILSEEDS=
for (( SEED=$BASESEED; SEED<$(($BASESEED+$RUNCOUNT)); SEED++ )); do
echo -n "Running test $BASENAME with random Seed $SEED" echo -n "Running test $BASENAME with random Seed $SEED"
# run simulation # run simulation
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.log & java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$SEED > $BASENAME.$SEED.coojalog &
JPID=$! JPID=$!
# Copy the log and only print "." if it changed # Copy the log and only print "." if it changed
touch $BASENAME.log.prog touch progress.log
while kill -0 $JPID 2> /dev/null while kill -0 $JPID 2> /dev/null
do do
sleep 1 sleep 1
diff $BASENAME.log $BASENAME.log.prog > /dev/null diff $BASENAME.$SEED.coojalog progress.log > /dev/null
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then
echo -n "." echo -n "."
cp $BASENAME.log $BASENAME.log.prog cp $BASENAME.$SEED.coojalog progress.log
fi fi
done done
rm $BASENAME.log.prog rm progress.log
# wait for end of simulation # wait for end of simulation
wait $JPID wait $JPID
JRV=$? JRV=$?
# Save testlog
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.$SEED.scriptlog
rm COOJA.log
TESTCOUNT+=1 TESTCOUNT+=1
if [ $JRV -eq 0 ] ; then if [ $JRV -eq 0 ] ; then
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog
OKCOUNT+=1 OKCOUNT+=1
echo " OK" echo " OK"
else else
FAILSEEDS+=" $BASESEED"
echo " FAIL ಠ_ಠ"
# Verbose output when using CI # Verbose output when using CI
if [ "$CI" = "true" ]; then if [ "$CI" = "true" ]; then
echo "==== $BASENAME.log ====" ; cat $BASENAME.log; echo "==== $BASENAME.$SEED.coojalog ====" ; cat $BASENAME.$SEED.coojalog;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog; echo "==== $BASENAME.$SEED.scriptlog ====" ; cat $BASENAME.$SEED.scriptlog;
echo "==== Files used for simulation (sha1sum) ===="
grep "Loading firmware from:" COOJA.log | cut -d " " -f 10 | uniq | xargs -r sha1sum
grep "Creating core communicator between Java class" COOJA.log | cut -d " " -f 17 | uniq | xargs -r sha1sum
else else
tail -50 $BASENAME.log ; echo "==== Check $BASENAME.$SEED.coojalog and $BASENAME.$SEED.scriptlog for details ====";
fi; fi
mv COOJA.testlog $BASENAME.$SEED.faillog
echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$SEED.faillog;
fi fi
shift
done done
echo "Test $BASENAME, successfull runs: $OKCOUNT/$TESTCOUNT"
if [ $TESTCOUNT -ne $OKCOUNT ] ; then if [ $TESTCOUNT -ne $OKCOUNT ] ; then
# At least one test failed # At least one test failed
touch COOJA.testlog; echo "$BASENAME: TEST FAIL ಠ_ಠ ($OKCOUNT/$TESTCOUNT, failed seeds:$FAILSEEDS)" > $BASENAME.testlog;
mv COOJA.testlog $BASENAME.testlog; else
echo "$BASENAME: TEST OK ($OKCOUNT/$TESTCOUNT)" > $BASENAME.testlog;
fi fi
# We do not want Make to stop -> Return 0 # We do not want Make to stop -> Return 0