import time, logging
from os.path import basename
from pymonetdb.exceptions import Error as DBError
def loadcsv_drop_pkey(conn, csvfile, q):
"""
"""
tabname = basename(csvfile)[:-4].lower()
pkey = tabname + "_id_pkey"
with open('loadcsv_drop_pkey.sql', 'r') as f:
qcsv = f.read()
params = {'tabname': tabname
,'pkey': pkey}
query = qcsv % (params)
#print "Query:\n%s" % (query)
try:
cursor = conn.cursor()
t0 = time.time()
cursor.execute(query)
t = round(1000 * (time.time() - t0), 3)
conn.commit()
print "# AL: (Q%s: %s ms) Altered table %s dropped pkey %s" % (q, t, tabname, pkey)
logging.info("L;I?;Q%s;T%s;N0" % (q, t))
cursor.close()
except DBError, dbe:
print "Failed for reason %s on query %s" % (dbe, query)
raise
def alter_tabname_add_pkey(conn, csvfile, q):
"""Set a primary key for the table
"""
tabname = basename(csvfile)[:-4].lower()
if tabname.startswith("source_"):
pkey = "id"
elif tabname.startswith("forcedsource_"):
pkey = "deepSourceId,scienceCcdExposureId"
elif tabname.startswith("object_"):
pkey = "deepSourceId"
elif tabname.startswith("objectfulloverlap_"):
return
with open('sql/alter_tabname_add_pkey.sql', 'r') as f:
qcsv = f.read()
params = {'tabname': tabname
,'pkey': pkey}
query = qcsv % (params)
#print "query = %s" % (query)
try:
cursor = conn.cursor()
t0 = time.time()
cursor.execute(query)
t = round(1000 * (time.time() - t0), 3)
conn.commit()
print "# AT: (Q%s: %s ms) Altered table %s added pkey" % (q, t, tabname)
logging.info("L;I0;Q%s;T%s;N0" % (q, t))
cursor.close()
except DBError, dbe:
print "Failed for reason %s on query %s" % (dbe, query)
raise
def load_csv_copyinto(conn, csvfile, q):
"""Use copy into to load the csv file
"""
tabname = basename(csvfile)[:-4].lower()
with open('sql/loadcsv_copyinto.sql', 'r') as f:
qcsv = f.read()
params = {'tabname': tabname
,'csvfile': csvfile}
query = qcsv % (params)
#print "Query:\n%s" % (query)
try:
cursor = conn.cursor()
t0 = time.time()
n = cursor.execute(query)
t = round(1000 * (time.time() - t0), 3)
conn.commit()
print "# CI: (Q%s: %s ms) Copied %s records into table %s" % (q, t, n, tabname)
logging.info("L;I0;Q%s;T%s;N%s" % (q, t, n))
cursor.close()
except DBError, dbe:
print "Failed for reason %s on query %s" % (dbe, query)
raise
def load_csv(conn, csvfile, q):
"""Load the csvfile, then set the primary key
"""
#loadcsv_drop_pkey(conn, csvfile, q+=?)
load_csv_copyinto(conn, csvfile, q)
alter_tabname_add_pkey(conn, csvfile, q+1)