diff --git a/hotloop.queries.sh b/hotloop.queries.sh new file mode 100755 index 0000000000000000000000000000000000000000..be315c8248f65ebb15e6674741f765bfb03e5ea0 --- /dev/null +++ b/hotloop.queries.sh @@ -0,0 +1,118 @@ +#/bin/bash +# +#+---------------------------------------------------------------------+ +#| | +#| Script to run LSST baseline queries in MonetDB in hot mode. | +#| | +#| This script assumes the database exists and is loaded with the | +#| 2.0 TB csv files kindly provided by F. Jammes. Run the python | +#| iscript load_lsst_data.py to load the database. | +#| | +#| The SQL file name tells corresponds to the query number, i.e. | +#| query = {q1, q2, ..., q13} | +#| | +#| The timing results are written to a log file with these variables | +#| in its name as well, including the node name. | +#| | +#| Then use the baq_node_results.py script to plot the timings. | +#| | +#| Run it as | +#| ./hotloop.queries.sh $dbname $dbversion $dbfarm $sqldir | +#| | +#| Bart Scheers (CWI, 2016) | +#+---------------------------------------------------------------------+ + +# We are in the "hot" loop script +temp=hot + +dbname=$1 +dbversion=$2 +dbfarm=$3 +sqldir=$4 + +if [ ! -d $sqldir ]; then + echo "Directory $sqldir does not exist." + exit 64 +fi +sqlfiles=($sqldir/q*.sql) + +logdir=${sqldir}/log +if [ ! -d $logdir ]; then + echo "Directory $logdir does not exist." + exit 64 +fi + +echo +echo "################:" +echo "# Running node : $( hostname -a )" +echo "# Using database: $dbname" +echo "# version: $dbversion" +echo "# which: $( which mserver5 )" +echo "# Using dbfarm : $dbfarm" +echo "# SQL dir : ${sqldir}" +echo "# Log dir : ${logdir}" +echo "################:" + +# Needed for stripping the mclient milliseconds timing strings +suffms="ms" + +# Here we iterate through the sqlfiles in the sqldir +for ((i=0; i<${#sqlfiles[@]}; i++)); do + sqlfile=${sqlfiles[$i]} + echo "Working on sqlfile: ${sqlfile}" + # get the file name + IFS='/' read -r -a qf <<< "$sqlfile" + # strip to the .sql extension + IFS='.' read -ra q <<< "${qf[-1]}" + query=${q[0]} + qlog=${logdir}/${query}.${temp}.$( hostname -a ).${dbversion}.log + echo "Logging query times in: ${qlog}" + + monetdb stop -a + monetdbd stop $dbfarm + echo "Stopping monetdbd..." + running="$( pgrep -u${USER} -l | grep monetdbd )" + sp='/-\|' + #printf ' ' + while [ -n "$running" ]; do + running="$( pgrep -u${USER} -l | grep monetdbd )" + #printf '\b%.1s' "$sp" + sp=${sp#?}${sp%???} + sleep 1 + done + echo "monetdbd stopped" + echo 3 | sudo tee /proc/sys/vm/drop_caches + #echo "UNFORTUNATELY: 'echo 3 | sudo tee /proc/sys/vm/drop_caches' DOES NOT WORK" + # We forward output to /dev/null, since mother script might not close + # when we use 2>&1 | tee log + monetdbd start $dbfarm > /dev/null 2>&1 + monetdb start $dbname + echo "monetdbd and monetdb started; $dbname running" + mclient -d$dbname -ftimer -ims < $sqlfile + echo "First query ran" + + START=1 + END=10 + echo "Running hot bulk associations ${END} times..." + + for (( c=$START; c<=$END; c++ )) + do + echo "${query}; run: $c" + res=($( mclient -d$dbname -ftimer -ims < $sqlfile)) + echo "${res[@]}" + + # strip the ms + tms=${res[-1]%$suffms} + + echo "${tms} milliseconds" + + if [ $c -eq 1 ]; then + echo "${c};${tms}" > $qlog + else + echo "${c};${tms}" >> $qlog + fi + + done +done + +echo "READY. Hot bulk associations ran ${END} times."