*.pyc

Running LSST baseline queries in MonetDB

========================================

See blog post ... for further details.

You need to have several LSST csv files (with artificial data), 

which can be requested from the LSST DM team.

MonetDB

-------

Run the installation script ``install_monetdb.sh``, 

start the daemon and create a database instance.

Loading

-------

The python load script ``load_lsst_data.py`` defines the 

database schema with table and partition defintions

and loads the csv files into the database.

Running queries

---------------

Two shell scripts run the queries in a hot and cold mode

``hotloop.queries.sh`` and ``coldloop.queries.sh``, resp.

The averages and standard deviations

of ten runs are reported in the log files

Plotting results

----------------

Use the python script ``plot_baselines.py`` to plot a bar chart 

of the results in comparison with the S15 MySQL results.

More info

---------

For more info send an e-mail to bscheers at cwi.nl.

# /bin/bash

#

#+---------------------------------------------------------------------+

#|                                                                     |

#| Script to run LSST baseline queries in MonetDB in hot mode.         |

#|                                                                     |

#| This script assumes the database exists and is loaded with the      |

#| 2.0 TB csv files kindly provided by F. Jammes. Run the python       |

#| iscript load_lsst_data.py to load the database.                     |

#|                                                                     |

#| The SQL file name tells corresponds to the query number, i.e.       |

#| query = {q1, q2, ..., q13}                                          |

#|                                                                     |

#| The timing results are written to a log file with these variables   |

#| in its name as well, including the node name.                       |

#|                                                                     |

#| Then use the baq_node_results.py script to plot the timings.        |

#|                                                                     |

#| Run it as                                                           |

#| ./hotloop.queries.sh $dbname $dbversion $dbfarm $sqldir             |

#|                                                                     |

#|                                            Bart Scheers (CWI, 2016) |

#+---------------------------------------------------------------------+

# We are in the "cold" loop script

temp=cold

dbname=$1

dbversion=$2

dbfarm=$3

sqldir=$4

if [ ! -d $sqldir ]; then

    echo "Directory $sqldir does not exist."

    exit 64

fi

sqlfiles=($sqldir/q*.sql)

logdir=${sqldir}/log

if [ ! -d $logdir ]; then

    echo "Directory $logdir does not exist."

    exit 64

fi

echo

echo "################:"

echo "# Running node  : $( hostname -a )"

echo "# Using database: $dbname"

echo "#        version: $dbversion"

echo "#          which: $( which mserver5 )"

echo "# Using dbfarm  : $dbfarm"

echo "# SQL dir       : ${sqldir}"

echo "# Log dir       : ${logdir}"

echo "################:"

# Needed for stripping the mclient milliseconds timing strings

suffms="ms"

# Here we iterate through the sqlfiles in the sqldir

for ((i=0; i<${#sqlfiles[@]}; i++)); do

    sqlfile=${sqlfiles[$i]}

    echo "Working on sqlfile: ${sqlfile}"

    # get the file name

    IFS='/' read -r -a qf <<< "$sqlfile"

    # strip to the .sql extension

    IFS='.' read -ra q <<< "${qf[-1]}"

    query=${q[0]}

    qlog=${logdir}/${query}.${temp}.$( hostname -a ).${dbversion}.log

    echo "Logging query times in: ${qlog}"

    START=1

    END=10

    echo "Running cold bulk associations ${END} times..."

    for (( c=$START; c<=$END; c++ ))

    do

        echo "Run $c"

        monetdb stop -a 

        monetdbd stop $dbfarm

        echo "Stopping monetdbd..."

        running="$( pgrep -u${USER} -l | grep monetdbd )"

        sp='/-\|'

        #printf ' '

        while [ -n "$running" ]; do

            running="$( pgrep -u${USER} -l | grep monetdbd )"

            #printf '\b%.1s' "$sp"

            sp=${sp#?}${sp%???}

            sleep 1

        done

        echo "monetdbd stopped"

        echo 3 | sudo tee /proc/sys/vm/drop_caches

        #echo "UNFORTUNATELY: 'echo 3 | sudo tee /proc/sys/vm/drop_caches' DOES NOT WORK"

        # We forward output to /dev/null, since mother script might not close

        # when we use 2>&1 | tee log

        monetdbd start $dbfarm > /dev/null 2>&1

        monetdb start $dbname

        echo "monetdb and $dbname started"

        echo "Cold query run time:"

        res=($( mclient -d$dbname -ftimer -ims < $sqlfile))

        echo "${res[@]}"

        # strip the ms

        tms=${res[-1]%$suffms}

        echo "${tms} milliseconds"

        if [ $c -eq 1 ]; then

            echo "${c};${tms}" > $qlog

        else

            echo "${c};${tms}" >> $qlog

        fi

    done

done

echo "READY. Cold bulk associations ran ${END} times."

#/bin/bash

#

#+---------------------------------------------------------------------+

#|                                                                     |

#| Script to run LSST baseline queries in MonetDB in hot mode.         |

#|                                                                     |

#| This script assumes the database exists and is loaded with the      |

#| 2.0 TB csv files kindly provided by F. Jammes. Run the python       |

#| iscript load_lsst_data.py to load the database.                     |

#|                                                                     |

#| The SQL file name tells corresponds to the query number, i.e.       |

#| query = {q1, q2, ..., q13}                                          |

#|                                                                     |

#| The timing results are written to a log file with these variables   |

#| in its name as well, including the node name.                       |

#|                                                                     |

#| Then use the baq_node_results.py script to plot the timings.        |

#|                                                                     |

#| Run it as                                                           |

#| ./hotloop.queries.sh $dbname $dbversion $dbfarm $sqldir             |

#|                                                                     |

#|                                            Bart Scheers (CWI, 2016) |

#+---------------------------------------------------------------------+

# We are in the "hot" loop script

temp=hot

dbname=$1

dbversion=$2

dbfarm=$3

sqldir=$4

if [ ! -d $sqldir ]; then

    echo "Directory $sqldir does not exist."

    exit 64

fi

sqlfiles=($sqldir/q*.sql)

logdir=${sqldir}/log

if [ ! -d $logdir ]; then

    echo "Directory $logdir does not exist."

    exit 64

fi

echo

echo "################:"

echo "# Running node  : $( hostname -a )"

echo "# Using database: $dbname"

echo "#        version: $dbversion"

echo "#          which: $( which mserver5 )"

echo "# Using dbfarm  : $dbfarm"

echo "# SQL dir       : ${sqldir}"

echo "# Log dir       : ${logdir}"

echo "################:"

# Needed for stripping the mclient milliseconds timing strings

suffms="ms"

# Here we iterate through the sqlfiles in the sqldir

for ((i=0; i<${#sqlfiles[@]}; i++)); do

    sqlfile=${sqlfiles[$i]}

    echo "Working on sqlfile: ${sqlfile}"

    # get the file name

    IFS='/' read -r -a qf <<< "$sqlfile"

    # strip to the .sql extension

    IFS='.' read -ra q <<< "${qf[-1]}"

    query=${q[0]}

    qlog=${logdir}/${query}.${temp}.$( hostname -a ).${dbversion}.log

    echo "Logging query times in: ${qlog}"

    monetdb stop -a 

    monetdbd stop $dbfarm

    echo "Stopping monetdbd..."

    running="$( pgrep -u${USER} -l | grep monetdbd )"

    sp='/-\|'

    #printf ' '

    while [ -n "$running" ]; do

        running="$( pgrep -u${USER} -l | grep monetdbd )"

        #printf '\b%.1s' "$sp"

        sp=${sp#?}${sp%???}

        sleep 1

    done

    echo "monetdbd stopped"

    echo 3 | sudo tee /proc/sys/vm/drop_caches

    #echo "UNFORTUNATELY: 'echo 3 | sudo tee /proc/sys/vm/drop_caches' DOES NOT WORK"

    # We forward output to /dev/null, since mother script might not close

    # when we use 2>&1 | tee log

    monetdbd start $dbfarm > /dev/null 2>&1

    monetdb start $dbname

    echo "monetdbd and monetdb started; $dbname running"

    mclient -d$dbname -ftimer -ims < $sqlfile

    echo "First query ran"

    START=1

    END=10

    echo "Running hot bulk associations ${END} times..."

    for (( c=$START; c<=$END; c++ ))

    do

        echo "${query}; run: $c"

        res=($( mclient -d$dbname -ftimer -ims < $sqlfile))

        echo "${res[@]}"

        # strip the ms

        tms=${res[-1]%$suffms}

        echo "${tms} milliseconds"

        if [ $c -eq 1 ]; then

            echo "${c};${tms}" > $qlog

        else

            echo "${c};${tms}" >> $qlog

        fi

    done

done

echo "READY. Hot bulk associations ran ${END} times."

#! /bin/bash

function init_db {

    db=$1

    monetdb stop $db

    monetdb destroy -f $db

    monetdb create $db

    #monetdb set nthreads=1 $db

    monetdb set embedpy=yes $db

    monetdb release $db

    monetdb start $db

}

init_db $1

#!/bin/bash

################################################################################

#                                                                              #

# This script installs the specified MonetDB release.                          #

# Provide the script with the release name, version number and directory       #

# where the database will be installed.                                        #

#                                                                              #

# Run as (for example):                                                        #

# ./install_monetdb.sh Dec2016-SP5 11.25.23 /scratch/myname/databases          #

#                                                                              #

################################################################################

rname=$1

mname=${rname}

mversion=$2

mdbname=MonetDB-${mname}

mdbversion=MonetDB-${mversion}

dbdir=$3

mkdir -p $dbdir

cd $dbdir

rm -Rf $mdbname

# Used for releases:

wget http://www.monetdb.org/downloads/sources/${mname}/${mdbversion}.tar.bz2

# Used for test-releases:

#wget http://dev.monetdb.org/downloads/testing/sources/${rname}/${mdbversion}.tar.bz2

tar -xvjf ${mdbversion}.tar.bz2

mkdir -p $dbdir/$mdbname/build

cd $dbdir/$mdbname/build

# On Ubuntu this has to run with --disable-strict --disable-geom, others may use --enable-strict

# To copy the ubuntu environment we use these settings (although the latter are for Fedora)

#../$mdbversion/configure --prefix=$dbdir/$mdbname --disable-optimize --enable-debug --enable-assert --disable-strict --disable-geom 2>&1 | tee configure.output.log

../../$mdbversion/configure --prefix=$dbdir/$mdbname --enable-optimize --disable-debug --disable-assert --disable-strict --disable-geom --enable-fits 2>&1 | tee configure.output.log

make 2>&1 | tee make.output.log

make install

echo 

echo "Don't forget to set the PATHs in your ~/.bashrc"

echo "READY"

import os, sys, time, logging

import glob, socket, subprocess

from os.path import basename

from pymonetdb.sql.connections import Connection as connect

from pymonetdb.exceptions import Error as DBError

import lsst_schema_queries as sch

import lsst_queries as q

#######################################

def main():

    dbport = 53000

    dbrelease = "Dec2016"

    dbversion = "SP5"

    dbname = "lsst_napels"

    node = socket.gethostname().split('.')[0]

    logfile = 'log/' + node + "_lsst_" + dbrelease + '_' + dbversion + "_" + time.strftime("%Y%m%d-%H%M%S") + ".log"

    logging.basicConfig(filename=logfile, filemode='w', format='%(message)s', level=logging.INFO)

    logging.info('Started')

    print "DB (%s) logging in file %s" % (dbname, logfile)

    csvdir = "/export/scratch2/bscheers/lsst/csvs"

    csvfiles = sorted(glob.glob(csvdir + '/S*_*.csv'))

    print "len(csvfiles) = %s" % (len(csvfiles))

    p = subprocess.Popen(["./init_db.sh", dbname], stdout=subprocess.PIPE)

    output, err = p.communicate()

    print "O:",output

    if err is not None:

        print "E:",err

    hname = 'localhost'

    uname = 'monetdb'

    pword = 'monetdb'

    tstart = time.time()

    conn = connect(hostname=hname, port=dbport, username=uname, password=pword, database=dbname)

    fcnt = 0

    scnt, fscnt, ocnt, ofocnt = 0, 0, 0, 0

    for f in csvfiles:

        fcnt += 1

        t0 = time.time()

        tabname = basename(f)[:-4].lower()

        print "====>\n#### Loading CSV file nr %s: %s ####" % (fcnt, basename(f))

        if tabname.startswith("source_"):

            scnt += 1

            sch.create_table(conn, basename(f), q=1)

            q.load_csv(conn, f, q=2) # executes queries 2 & 3

            if scnt == 1:

                sch.create_merge_table(conn, basename(f), q=4)

                sch.alter_merge_table_add_pkey(conn, f, q=5)

            sch.alter_merge_table_add_table(conn, basename(f), q=6)

        elif tabname.startswith("forcedsource_"):

            fscnt += 1

            sch.create_table(conn, basename(f), q=7)

            q.load_csv(conn, f, q=8)

            if fscnt == 1:

                sch.create_merge_table(conn, basename(f), q=10)

                sch.alter_merge_table_add_pkey(conn, f, q=11)

            sch.alter_merge_table_add_table(conn, basename(f), q=12)

        elif tabname.startswith("object_"):

            ocnt += 1

            sch.create_table(conn, basename(f), q=13)

            q.load_csv(conn, f, q=14)

            if ocnt == 1:

                sch.create_merge_table(conn, basename(f), q=16)

                sch.alter_merge_table_add_pkey(conn, f, q=17)

            sch.alter_merge_table_add_table(conn, basename(f), q=18)

        elif tabname.startswith("objectfulloverlap_"):

            ofocnt += 1

            sch.create_table(conn, basename(f), q=19)

            q.load_csv(conn, f, q=20)

            if ofocnt == 1:

                sch.create_merge_table(conn, basename(f), q=22)

                sch.alter_merge_table_add_pkey(conn, f, q=23)

            sch.alter_merge_table_add_table(conn, basename(f), q=24)

        else:

            sys.exit("No valid tabname: %s" % (tabname))

        print "# L : Total copy-into time for this file: %s seconds" % (str(round(time.time() - t0, 3)))

        print "# L : Total processing time: %s seconds" % (str(round(time.time() - tstart, 3)))

    conn.close()

    print "# DB: Connection closed"

    print "\n#### Total elapsed time: %s seconds ####\n" % (str(round(time.time() - tstart, 3)))

    logging.info('Ended')

    print "Check log file: %s" % (logfile)

if __name__ == '__main__':

    main()

# Ignore everything in this directory

*

# Except this file

!.gitignore

import time, logging

from os.path import basename

from pymonetdb.exceptions import Error as DBError

def loadcsv_drop_pkey(conn, csvfile, q):

    """

    """

    tabname = basename(csvfile)[:-4].lower()

    pkey = tabname + "_id_pkey"

    with open('loadcsv_drop_pkey.sql', 'r') as f:

        qcsv = f.read()

    params = {'tabname': tabname

             ,'pkey': pkey}

    query = qcsv % (params)

    #print "Query:\n%s" % (query)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        print "# AL: (Q%s: %s ms)  Altered table %s dropped pkey %s" % (q, t, tabname, pkey)

        logging.info("L;I?;Q%s;T%s;N0" % (q, t))

        cursor.close()

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def alter_tabname_add_pkey(conn, csvfile, q):

    """Set a primary key for the table

    """

    tabname = basename(csvfile)[:-4].lower()

    if tabname.startswith("source_"):

        pkey = "id"

    elif tabname.startswith("forcedsource_"):

        pkey = "deepSourceId,scienceCcdExposureId"

    elif tabname.startswith("object_"):

        pkey = "deepSourceId"

    elif tabname.startswith("objectfulloverlap_"):

        return

    with open('sql/alter_tabname_add_pkey.sql', 'r') as f:

        qcsv = f.read()

    params = {'tabname': tabname

             ,'pkey': pkey}

    query = qcsv % (params)

    #print "query = %s" % (query)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        print "# AT: (Q%s: %s ms) Altered table %s added pkey" % (q, t, tabname)

        logging.info("L;I0;Q%s;T%s;N0" % (q, t))

        cursor.close()

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def load_csv_copyinto(conn, csvfile, q):

    """Use copy into to load the csv file

    """

    tabname = basename(csvfile)[:-4].lower()

    with open('sql/loadcsv_copyinto.sql', 'r') as f:

        qcsv = f.read()

    params = {'tabname': tabname

             ,'csvfile': csvfile}

    query = qcsv % (params)

    #print "Query:\n%s" % (query)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        n = cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        print "# CI: (Q%s: %s ms) Copied %s records into table %s" % (q, t, n, tabname)

        logging.info("L;I0;Q%s;T%s;N%s" % (q, t, n))

        cursor.close()

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def load_csv(conn, csvfile, q):

    """Load the csvfile, then set the primary key

    """

    #loadcsv_drop_pkey(conn, csvfile, q+=?)

    load_csv_copyinto(conn, csvfile, q)

    alter_tabname_add_pkey(conn, csvfile, q+1)

import sys, time, logging, subprocess

from os.path import basename

from pymonetdb.sql.connections import Connection as connect

from pymonetdb.exceptions import Error as DBError

def create_table(conn, csvfile, q):

    """General function to create either a source, forcedsource,

    object or objectfulloverlap table. The corresponding sql is

    executed.

    """

    tabname = basename(csvfile)[:-4].lower()

    fname = tabname.split("_")[0]

    sqlfile = "sql/create_" + fname + ".sql"

    with open(sqlfile, 'r') as f:

        qu = f.read()

    params = {'tabname': tabname}

    query = qu % (params)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        cursor.close()

        print "# CT: (Q%s: %s ms) Created table: %s" % (q, t, tabname)

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def create_merge_table(conn, csvfile, q):

    """General function to create a merge table

    """

    tabname = basename(csvfile)[:-4].lower()

    merge_tabname = tabname.split("_")[0]

    with open('sql/create_merge_tabname.sql', 'r') as f:

        qu = f.read()

    params = {'merge_tabname': merge_tabname

             ,'tabname': tabname}

    query = qu % (params)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        cursor.close()

        print "# CM: (Q%s: %s ms) Created merge table %s based upon: %s" \

            % (q, t, merge_tabname, tabname)

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def alter_merge_table_add_table(conn, csvfile, q):

    """Add a (sub)table to the merge table

    """

    tabname = basename(csvfile)[:-4].lower()

    merge_tabname = tabname.split("_")[0]

    with open('sql/alter_merge_table_add_table.sql', 'r') as f:

        qu = f.read()

    params = {'merge_tabname': merge_tabname

             ,'tabname': tabname}

    query = qu % (params)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        cursor.close()

        print "# AM: (Q%s: %s ms) Alter merge table %s add table %s" \

            % (q, t, merge_tabname, tabname)

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

def alter_merge_table_add_pkey(conn, csvfile, q):

    """Add a primary key to the merge table

    """

    tabname = basename(csvfile)[:-4].lower()

    merge_tabname = tabname.split("_")[0]

    if tabname.startswith("source_"):

        pkey = "id"

    elif tabname.startswith("forcedsource_"):

        pkey = "deepSourceId,scienceCcdExposureId"

    elif tabname.startswith("object_"):

        pkey = "deepSourceId"

    elif tabname.startswith("objectfulloverlap_"):

        return

    # NOTE: this sql is being used in load csv part as well (lsst_queries.py)

    with open('sql/alter_tabname_add_pkey.sql', 'r') as f:

        qu = f.read()

    params = {'tabname': merge_tabname

             ,'pkey': pkey}

    query = qu % (params)

    try:

        cursor = conn.cursor()

        t0 = time.time()

        cursor.execute(query)

        t = round(1000 * (time.time() - t0), 3)

        conn.commit()

        cursor.close()

        print "# AM: (Q%s: %s ms) Alter merge table %s add primary key (%s)" \

            % (q, t, merge_tabname, pkey)

    except DBError, dbe:

        print "Failed for reason %s on query %s" % (dbe, query)

        raise

#!/usr/bin/python

import matplotlib as mpl

mpl.use('Agg')

import os, errno, time, sys, pylab

from scipy import *

from scipy import optimize

from scipy import special

from scipy import stats

#import numpy as np

import matplotlib.cm as cm

import matplotlib.ticker as ticker

from datetime import datetime

import logging, csv

from os.path import basename, exists

plotdir = os.path.abspath('.') + '/results'

if not os.path.exists(plotdir):

    os.makedirs(plotdir)

    print "Created plotdir: %s" % (plotdir)

logdir = os.path.abspath('.') + '/log'

def stdev(v):

    """

    Return the Bessel corrected standard deviation of a series of measurements

    """

    return sqrt(len(v) * (mean(square(v)) - mean(v)**2) / (len(v) - 1))

def mysql_querytimes():

    """

    """

    # The s15 mysql query timing results taken from

    # https://confluence.lsstcorp.org/display/DM/S15+Large+Scale+Tests

    # Values on website are reported in seconds

    counts = [47.75, 40.99, 48.33]

    shortrunning = [0.09, 0.33]

    fullscans = [247.61, 244.24, 1088.09, 1077.38, 898.61]

    joins = [1381.44, 1291.38]

    neighbor = [676.02]

    mysqls15 = asarray(counts + shortrunning + fullscans + joins + neighbor)

    #tms = [ts * 1000 for ts in mysqls15]

    return mysqls15

def monetdb_querytimes(qlog):

    """

    """

    # From the qlog name, we can deduce the query number

    # and run type

    qtime = []

    with open(qlog, 'rb') as csvfile:

        qreader = csv.reader(csvfile, delimiter=';')

        for row in qreader:

            run = int(row[0])

            qtime.append(float(row[1])) # unit is ms

    #for i in range(len(qtime)):

    #    print "%s: qtime = %s" % (i, qtime[i])

    return mean(qtime), stdev(qtime)

def plot_results():

    """

    """

    logdir = 'sql/s15/log'

    plotdir = '.'

    plotfiles = []

    node = "stones10"

    dbversion = "dec2016sp1"

    # We have hot and cold runs

    runtype = ['hot', 'cold']

    #runtype = ['hot']

    # We divide the queries in five groups

    counts = ['q01', 'q02', 'q03']

    shortrunning = ['q04', 'q05']

    fullscans = ['q06', 'q07', 'q08', 'q09', 'q10']

    joins = ['q11', 'q12']

    neighbor = ['q13']

    queries = asarray(counts + shortrunning + fullscans + joins + neighbor)

    mysqls15 = mysql_querytimes()

    monetdbs15 = {}

    for temp in runtype:

        monetdbs15[temp] = {}

        monetdbs15[temp]['qt_avg'] = []

        monetdbs15[temp]['qt_std'] = []

        for qnr in queries:

            qlog = logdir + "/%s.%s.%s.%s.log" % (qnr, temp, node, dbversion)

            #print "qlog = %s" % (qlog)

            if not exists(qlog):

                sys.exit("Run timings. Log file %s does not exist (yet)." % (qlog))

            qt_avg, qt_std = monetdb_querytimes(qlog)

            print "%s qnr: %s: qt_avg = %s; qt_std = %s" % (temp, qnr, qt_avg, qt_std)

            monetdbs15[temp]['qt_avg'].append(qt_avg)

            monetdbs15[temp]['qt_std'].append(qt_std)

    mhot_t = [t * 0.001 for t in monetdbs15['hot']['qt_avg']]

    mhot_s = [s * 0.001 for s in monetdbs15['hot']['qt_std']]

    mcold_t = [t * 0.001 for t in monetdbs15['cold']['qt_avg']]

    mcold_s = [s * 0.001 for s in monetdbs15['cold']['qt_std']]

    #print "monetdbs15 = %s" % (monetdbs15)

    fig = pylab.figure(figsize=(28,8))

    ax = fig.add_subplot(111)

    width = 0.2

    ekw = dict(ecolor='k', lw=2, capsize=5, capthick=2)

    r1 = ax.bar(arange(len(mysqls15)), mysqls15, width, color='gray' \

               ,error_kw=ekw,label='MySQL')

    r2 = ax.bar(arange(len(mcold_t)) + width, mcold_t, width, yerr=mcold_s, color='dodgerblue' \

               ,error_kw=ekw,label='MonetDB cold')

    r3 = ax.bar(arange(len(mhot_t)) + 2 * width, mhot_t, width, yerr=mhot_s, color='red' \

               ,error_kw=ekw,label='MonetDB hot')

    ax.set_ylabel('Time [s]', fontsize='25')

    ax.set_xticks(arange(len(mysqls15)) + width)

    ax.set_xticklabels([q.upper() for q in queries])

    #ax.set_ylim(ymin=10, ymax=35000)

    ax.set_ylim(ymin=0.001, ymax=10000)

    ax.set_yscale("log")

    #ax.legend(loc='upper right')

    ax.legend(loc='upper left', fontsize='25')

    for i in range(len(ax.get_xticklabels())):

        #ax.get_xticklabels()[i].set_size('xx-large')

        ax.get_xticklabels()[i].set_size('25')

    for i in range(len(ax.get_yticklabels())):

        #ax.get_yticklabels()[i].set_size('xx-large')

        ax.get_yticklabels()[i].set_size('25')

    pylab.grid(True)

    fname = 'baseline_lsst_queries.eps'

    plotfile = plotdir + '/' + fname

    plotfiles.append(plotfile)

    pylab.savefig(plotfile, dpi=400, bbox_inches='tight')

    print plotfile

    return plotfile

def main():

    #query = sys.argv[1]

    plot_results()

if __name__ == '__main__':

    main()

ALTER TABLE %(merge_tabname)s ADD TABLE %(tabname)s;

ALTER TABLE %(tabname)s ADD PRIMARY KEY (%(pkey)s);

CREATE TABLE %(tabname)s 

  (deepSourceId BIGINT NOT NULL

  ,scienceCcdExposureId BIGINT NOT NULL

  ,psfFlux float DEFAULT NULL

  ,psfFluxSigma float DEFAULT NULL

  ,flagBadMeasCentroid BOOLEAN NOT NULL

  ,flagPixEdge BOOLEAN NOT NULL

  ,flagPixInterpAny BOOLEAN NOT NULL