File diff 000000000000 → 72890999e24d
lsst_schema_queries.py
Show inline comments
 
new file 100644
 
import sys, time, logging, subprocess
 
from os.path import basename
 

	
 
from pymonetdb.sql.connections import Connection as connect
 
from pymonetdb.exceptions import Error as DBError
 

	
 
def create_table(conn, csvfile, q):
 
    """General function to create either a source, forcedsource,
 
    object or objectfulloverlap table. The corresponding sql is
 
    executed.
 
    """
 

	
 
    tabname = basename(csvfile)[:-4].lower()
 
    fname = tabname.split("_")[0]
 
    sqlfile = "sql/create_" + fname + ".sql"
 
    with open(sqlfile, 'r') as f:
 
        qu = f.read()
 
    params = {'tabname': tabname}
 
    query = qu % (params)
 

	
 
    try:
 
        cursor = conn.cursor()
 
        t0 = time.time()
 
        cursor.execute(query)
 
        t = round(1000 * (time.time() - t0), 3)
 
        conn.commit()
 
        cursor.close()
 
        print "# CT: (Q%s: %s ms) Created table: %s" % (q, t, tabname)
 
    except DBError, dbe:
 
        print "Failed for reason %s on query %s" % (dbe, query)
 
        raise
 

	
 
def create_merge_table(conn, csvfile, q):
 
    """General function to create a merge table
 
    """
 

	
 
    tabname = basename(csvfile)[:-4].lower()
 
    merge_tabname = tabname.split("_")[0]
 
    with open('sql/create_merge_tabname.sql', 'r') as f:
 
        qu = f.read()
 
    params = {'merge_tabname': merge_tabname
 
             ,'tabname': tabname}
 
    query = qu % (params)
 

	
 
    try:
 
        cursor = conn.cursor()
 
        t0 = time.time()
 
        cursor.execute(query)
 
        t = round(1000 * (time.time() - t0), 3)
 
        conn.commit()
 
        cursor.close()
 
        print "# CM: (Q%s: %s ms) Created merge table %s based upon: %s" \
 
            % (q, t, merge_tabname, tabname)
 
    except DBError, dbe:
 
        print "Failed for reason %s on query %s" % (dbe, query)
 
        raise
 

	
 
def alter_merge_table_add_table(conn, csvfile, q):
 
    """Add a (sub)table to the merge table
 
    """
 

	
 
    tabname = basename(csvfile)[:-4].lower()
 
    merge_tabname = tabname.split("_")[0]
 
    with open('sql/alter_merge_table_add_table.sql', 'r') as f:
 
        qu = f.read()
 
    params = {'merge_tabname': merge_tabname
 
             ,'tabname': tabname}
 
    query = qu % (params)
 

	
 
    try:
 
        cursor = conn.cursor()
 
        t0 = time.time()
 
        cursor.execute(query)
 
        t = round(1000 * (time.time() - t0), 3)
 
        conn.commit()
 
        cursor.close()
 
        print "# AM: (Q%s: %s ms) Alter merge table %s add table %s" \
 
            % (q, t, merge_tabname, tabname)
 
    except DBError, dbe:
 
        print "Failed for reason %s on query %s" % (dbe, query)
 
        raise
 

	
 
def alter_merge_table_add_pkey(conn, csvfile, q):
 
    """Add a primary key to the merge table
 
    """
 

	
 
    tabname = basename(csvfile)[:-4].lower()
 
    merge_tabname = tabname.split("_")[0]
 
    if tabname.startswith("source_"):
 
        pkey = "id"
 
    elif tabname.startswith("forcedsource_"):
 
        pkey = "deepSourceId,scienceCcdExposureId"
 
    elif tabname.startswith("object_"):
 
        pkey = "deepSourceId"
 
    elif tabname.startswith("objectfulloverlap_"):
 
        return
 
    # NOTE: this sql is being used in load csv part as well (lsst_queries.py)
 
    with open('sql/alter_tabname_add_pkey.sql', 'r') as f:
 
        qu = f.read()
 
    params = {'tabname': merge_tabname
 
             ,'pkey': pkey}
 
    query = qu % (params)
 

	
 
    try:
 
        cursor = conn.cursor()
 
        t0 = time.time()
 
        cursor.execute(query)
 
        t = round(1000 * (time.time() - t0), 3)
 
        conn.commit()
 
        cursor.close()
 
        print "# AM: (Q%s: %s ms) Alter merge table %s add primary key (%s)" \
 
            % (q, t, merge_tabname, pkey)
 
    except DBError, dbe:
 
        print "Failed for reason %s on query %s" % (dbe, query)
 
        raise