|
new file 100755
|
|
|
#/bin/bash
|
|
|
#
|
|
|
#+---------------------------------------------------------------------+
|
|
|
#| |
|
|
|
#| Script to run LSST baseline queries in MonetDB in hot mode. |
|
|
|
#| |
|
|
|
#| This script assumes the database exists and is loaded with the |
|
|
|
#| 2.0 TB csv files kindly provided by F. Jammes. Run the python |
|
|
|
#| iscript load_lsst_data.py to load the database. |
|
|
|
#| |
|
|
|
#| The SQL file name tells corresponds to the query number, i.e. |
|
|
|
#| query = {q1, q2, ..., q13} |
|
|
|
#| |
|
|
|
#| The timing results are written to a log file with these variables |
|
|
|
#| in its name as well, including the node name. |
|
|
|
#| |
|
|
|
#| Then use the baq_node_results.py script to plot the timings. |
|
|
|
#| |
|
|
|
#| Run it as |
|
|
|
#| ./hotloop.queries.sh $dbname $dbversion $dbfarm $sqldir |
|
|
|
#| |
|
|
|
#| Bart Scheers (CWI, 2016) |
|
|
|
#+---------------------------------------------------------------------+
|
|
|
|
|
|
# We are in the "hot" loop script
|
|
|
temp=hot
|
|
|
|
|
|
dbname=$1
|
|
|
dbversion=$2
|
|
|
dbfarm=$3
|
|
|
sqldir=$4
|
|
|
|
|
|
if [ ! -d $sqldir ]; then
|
|
|
echo "Directory $sqldir does not exist."
|
|
|
exit 64
|
|
|
fi
|
|
|
sqlfiles=($sqldir/q*.sql)
|
|
|
|
|
|
logdir=${sqldir}/log
|
|
|
if [ ! -d $logdir ]; then
|
|
|
echo "Directory $logdir does not exist."
|
|
|
exit 64
|
|
|
fi
|
|
|
|
|
|
echo
|
|
|
echo "################:"
|
|
|
echo "# Running node : $( hostname -a )"
|
|
|
echo "# Using database: $dbname"
|
|
|
echo "# version: $dbversion"
|
|
|
echo "# which: $( which mserver5 )"
|
|
|
echo "# Using dbfarm : $dbfarm"
|
|
|
echo "# SQL dir : ${sqldir}"
|
|
|
echo "# Log dir : ${logdir}"
|
|
|
echo "################:"
|
|
|
|
|
|
# Needed for stripping the mclient milliseconds timing strings
|
|
|
suffms="ms"
|
|
|
|
|
|
# Here we iterate through the sqlfiles in the sqldir
|
|
|
for ((i=0; i<${#sqlfiles[@]}; i++)); do
|
|
|
sqlfile=${sqlfiles[$i]}
|
|
|
echo "Working on sqlfile: ${sqlfile}"
|
|
|
# get the file name
|
|
|
IFS='/' read -r -a qf <<< "$sqlfile"
|
|
|
# strip to the .sql extension
|
|
|
IFS='.' read -ra q <<< "${qf[-1]}"
|
|
|
query=${q[0]}
|
|
|
qlog=${logdir}/${query}.${temp}.$( hostname -a ).${dbversion}.log
|
|
|
echo "Logging query times in: ${qlog}"
|
|
|
|
|
|
monetdb stop -a
|
|
|
monetdbd stop $dbfarm
|
|
|
echo "Stopping monetdbd..."
|
|
|
running="$( pgrep -u${USER} -l | grep monetdbd )"
|
|
|
sp='/-\|'
|
|
|
#printf ' '
|
|
|
while [ -n "$running" ]; do
|
|
|
running="$( pgrep -u${USER} -l | grep monetdbd )"
|
|
|
#printf '\b%.1s' "$sp"
|
|
|
sp=${sp#?}${sp%???}
|
|
|
sleep 1
|
|
|
done
|
|
|
echo "monetdbd stopped"
|
|
|
echo 3 | sudo tee /proc/sys/vm/drop_caches
|
|
|
#echo "UNFORTUNATELY: 'echo 3 | sudo tee /proc/sys/vm/drop_caches' DOES NOT WORK"
|
|
|
# We forward output to /dev/null, since mother script might not close
|
|
|
# when we use 2>&1 | tee log
|
|
|
monetdbd start $dbfarm > /dev/null 2>&1
|
|
|
monetdb start $dbname
|
|
|
echo "monetdbd and monetdb started; $dbname running"
|
|
|
mclient -d$dbname -ftimer -ims < $sqlfile
|
|
|
echo "First query ran"
|
|
|
|
|
|
START=1
|
|
|
END=10
|
|
|
echo "Running hot bulk associations ${END} times..."
|
|
|
|
|
|
for (( c=$START; c<=$END; c++ ))
|
|
|
do
|
|
|
echo "${query}; run: $c"
|
|
|
res=($( mclient -d$dbname -ftimer -ims < $sqlfile))
|
|
|
echo "${res[@]}"
|
|
|
|
|
|
# strip the ms
|
|
|
tms=${res[-1]%$suffms}
|
|
|
|
|
|
echo "${tms} milliseconds"
|
|
|
|
|
|
if [ $c -eq 1 ]; then
|
|
|
echo "${c};${tms}" > $qlog
|
|
|
else
|
|
|
echo "${c};${tms}" >> $qlog
|
|
|
fi
|
|
|
|
|
|
done
|
|
|
done
|
|
|
|
|
|
echo "READY. Hot bulk associations ran ${END} times."
|