giomba/nes-proj
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support running multiple random seeds in case one fails

Browse Source
This commit is contained in:
Moritz 'Morty' Strübe 2014-10-08 09:22:37 +02:00
parent c0783e2841
commit 74b741bad2
2 changed files with 68 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
regression-tests/Makefile.simulation-test
View File

@ -30,8 +30,8 @@ TESTS=$(wildcard ??-*.csc)
TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS)) TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
LOGS=$(patsubst %.csc,%.log,$(TESTS)) LOGS=$(patsubst %.csc,%.log,$(TESTS))
FAILLOGS=$(patsubst %.csc,%.faillog,$(TESTS)) FAILLOGS=$(patsubst %.csc,%.faillog,$(TESTS))
#Set random seed to create reproduceable results. #Set random seeds to create reproduceable results.
RANDOMSEED=1 RANDOMSEED=1 5
CONTIKI=../.. CONTIKI=../..
@ -57,7 +57,7 @@ RUNALL=false
endif endif
%.testlog: %.csc cooja %.testlog: %.csc cooja
@$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" "$(RANDOMSEED)" @$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
clean: clean:
@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \ @rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \

114
regression-tests/simexec.sh
View File

@ -1,66 +1,82 @@
#!/bin/bash #!/bin/bash
# Do not return an error
RUNALL=$1 RUNALL=$1
CSC=$2 shift
CONTIKI=$3 # The simulation to run
BASENAME=$4 CSC=$1
RANDOMSEED=$5 shift
#Contiki directory
CONTIKI=$1
shift
#The basename of the experiment
BASENAME=$1
shift
# The test will end on the first successfull run
#set -x #set -x
echo -n "Running test $BASENAME " while (( "$#" )); do
RANDOMSEED=$1
echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log & java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
JPID=$! JPID=$!
# Copy the log and only print "." if it changed # Copy the log and only print "." if it changed
touch $BASENAME.log.prog touch $BASENAME.log.prog
while kill -0 $JPID 2> /dev/null while kill -0 $JPID 2> /dev/null
do do
sleep 1 sleep 1
diff $BASENAME.log $BASENAME.log.prog > /dev/null diff $BASENAME.log $BASENAME.log.prog > /dev/null
if [ $? -ne 0 ] if [ $? -ne 0 ]
then then
echo -n "." echo -n "."
cp $BASENAME.log $BASENAME.log.prog cp $BASENAME.log $BASENAME.log.prog
fi fi
done
rm $BASENAME.log.prog
wait $JPID
JRV=$?
if [ $JRV -eq 0 ] ; then
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog
echo " OK"
exit 0
fi
# In case of failure
#Verbose output when using CI
if [ "$CI" = "true" ]; then
echo "==== COOJA.log ====" ; cat COOJA.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
else
tail -50 COOJA.log ;
fi;
mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
shift
done done
rm $BASENAME.log.prog
#All seeds failed
wait $JPID echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
JRV=$?
if [ $JRV -eq 0 ] ; then
touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog
echo " OK"
exit 0
fi
# In case of failure
echo " FAIL ಠ_ಠ" | tee -a COOJA.testlog;
#Verbose output when using CI
if [ "$CI" = "true" ]; then
echo "==== COOJA.log ====" ; cat COOJA.log;
echo "==== COOJA.testlog ====" ; cat COOJA.testlog;
else
tail -50 COOJA.log ;
fi;
mv COOJA.testlog $BASENAME.faillog
# We do not want Make to stop -> Return 0 # We do not want Make to stop -> Return 0
if [ "$RUNALL" = "true" ] ; then if [ "$RUNALL" = "true" ] ; then
touch COOJA.testlog; touch COOJA.testlog;
mv COOJA.testlog $BASENAME.testlog; mv COOJA.testlog $BASENAME.testlog;
exit 0 exit 0
fi fi
#This is a failure
exit 1 exit 1