Files
@ 72890999e24d
Branch filter:
Location: DA/lsst_blog/coldloop.queries.sh - annotation
72890999e24d
3.9 KiB
text/x-sh
Running LSST baseline queries in MonetDB
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d 72890999e24d | # /bin/bash
#
#+---------------------------------------------------------------------+
#| |
#| Script to run LSST baseline queries in MonetDB in hot mode. |
#| |
#| This script assumes the database exists and is loaded with the |
#| 2.0 TB csv files kindly provided by F. Jammes. Run the python |
#| iscript load_lsst_data.py to load the database. |
#| |
#| The SQL file name tells corresponds to the query number, i.e. |
#| query = {q1, q2, ..., q13} |
#| |
#| The timing results are written to a log file with these variables |
#| in its name as well, including the node name. |
#| |
#| Then use the baq_node_results.py script to plot the timings. |
#| |
#| Run it as |
#| ./hotloop.queries.sh $dbname $dbversion $dbfarm $sqldir |
#| |
#| Bart Scheers (CWI, 2016) |
#+---------------------------------------------------------------------+
# We are in the "cold" loop script
temp=cold
dbname=$1
dbversion=$2
dbfarm=$3
sqldir=$4
if [ ! -d $sqldir ]; then
echo "Directory $sqldir does not exist."
exit 64
fi
sqlfiles=($sqldir/q*.sql)
logdir=${sqldir}/log
if [ ! -d $logdir ]; then
echo "Directory $logdir does not exist."
exit 64
fi
echo
echo "################:"
echo "# Running node : $( hostname -a )"
echo "# Using database: $dbname"
echo "# version: $dbversion"
echo "# which: $( which mserver5 )"
echo "# Using dbfarm : $dbfarm"
echo "# SQL dir : ${sqldir}"
echo "# Log dir : ${logdir}"
echo "################:"
# Needed for stripping the mclient milliseconds timing strings
suffms="ms"
# Here we iterate through the sqlfiles in the sqldir
for ((i=0; i<${#sqlfiles[@]}; i++)); do
sqlfile=${sqlfiles[$i]}
echo "Working on sqlfile: ${sqlfile}"
# get the file name
IFS='/' read -r -a qf <<< "$sqlfile"
# strip to the .sql extension
IFS='.' read -ra q <<< "${qf[-1]}"
query=${q[0]}
qlog=${logdir}/${query}.${temp}.$( hostname -a ).${dbversion}.log
echo "Logging query times in: ${qlog}"
START=1
END=10
echo "Running cold bulk associations ${END} times..."
for (( c=$START; c<=$END; c++ ))
do
echo "Run $c"
monetdb stop -a
monetdbd stop $dbfarm
echo "Stopping monetdbd..."
running="$( pgrep -u${USER} -l | grep monetdbd )"
sp='/-\|'
#printf ' '
while [ -n "$running" ]; do
running="$( pgrep -u${USER} -l | grep monetdbd )"
#printf '\b%.1s' "$sp"
sp=${sp#?}${sp%???}
sleep 1
done
echo "monetdbd stopped"
echo 3 | sudo tee /proc/sys/vm/drop_caches
#echo "UNFORTUNATELY: 'echo 3 | sudo tee /proc/sys/vm/drop_caches' DOES NOT WORK"
# We forward output to /dev/null, since mother script might not close
# when we use 2>&1 | tee log
monetdbd start $dbfarm > /dev/null 2>&1
monetdb start $dbname
echo "monetdb and $dbname started"
echo "Cold query run time:"
res=($( mclient -d$dbname -ftimer -ims < $sqlfile))
echo "${res[@]}"
# strip the ms
tms=${res[-1]%$suffms}
echo "${tms} milliseconds"
if [ $c -eq 1 ]; then
echo "${c};${tms}" > $qlog
else
echo "${c};${tms}" >> $qlog
fi
done
done
echo "READY. Cold bulk associations ran ${END} times."
|