diff --git a/django-dev.wsgi b/django-dev.wsgi index 52fa55f793ec369fb4b52a79620dafe42ab5c88b..dc709020a96392ea8b82322036012073ee4ec3b1 100644 --- a/django-dev.wsgi +++ b/django-dev.wsgi @@ -6,6 +6,7 @@ sys.path.append('/home/lars/django') # OK, the lib/lib64 situation should be handled better. sys.path.append('/opt/lscsoft-bleed/lib64/python2.4/site-packages') +sys.path.append('/opt/lscsoft-bleed/glue/lib64/python2.4/site-packages') sys.path.append('/opt/lscsoft-bleed/lib/python2.4/site-packages') sys.path.append('/opt/lscsoft/pylal/lib/python2.4/site-packages') @@ -16,3 +17,5 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'gracedb.settings_dev' import django.core.handlers.wsgi application = django.core.handlers.wsgi.WSGIHandler() +#os.environ['PKG_CONFIG_PATH'] = "${HOME}/lib/pkgconfig:${PKG_CONFIG_PATH}" + diff --git a/gracedb/models.py b/gracedb/models.py index 0e278e6ce74ca510887bb69ea57db78a23a0e12a..d42956120ffab7bfd0b9597215c91904051e8f5b 100644 --- a/gracedb/models.py +++ b/gracedb/models.py @@ -4,6 +4,8 @@ import thread import string import os +from gracedb.ligolw.models import CoincEvent + class User(models.Model): name = models.CharField(max_length=100) email = models.EmailField() @@ -45,6 +47,7 @@ class Event(models.Model): instruments = models.CharField(max_length=20, default="") nevents = models.PositiveIntegerField(null=True) likelihood = models.FloatField(null=True) + coincEvent = models.ForeignKey(CoincEvent, null=True) # NOT from coinc_event, but so, so common. # Note that the semantics for this is different depending diff --git a/gracedb/translator.py b/gracedb/translator.py index f344bac87fa1ac305bb28a3b7c62c7bf6c548968..57ff12f940df8af83e64369374754649d4ddde39 100644 --- a/gracedb/translator.py +++ b/gracedb/translator.py @@ -1,7 +1,8 @@ -import os +import os, sys from models import EventLog +from subprocess import Popen, PIPE import glue, glue.ligolw.utils from glue.gracedb.utils import InspiralCoincDef @@ -15,6 +16,36 @@ from glue.gracedb.utils import populate_inspiral_tables, \ populate_coinc_tables, \ write_output_files +# Importing this messes with other ligolw table actions. +#from gracedb.ligolw.insert import insert_ligolw_tables +import gracedb.ligolw + +def insert_ligolw_tables(xml_filename): + #insert_ligolw_tables(django.db.connection, xml_filename) + prog = os.path.dirname(gracedb.ligolw.__file__) + prog = os.path.join(prog, "insert.py") + e = dict(os.environ) + ppath = e.get("PYTHONPATH") or "" + ppath = ppath.split(':') + ppath = ppath + sys.path + e['PYTHONPATH'] = ':'.join(ppath) + p = Popen( (prog, "root", "", "gracedb", xml_filename), stdout=PIPE, stderr=PIPE, env=e) + out = p.stdout.read() + err = p.stderr.read() + p.wait() + out += p.stdout.read() + if out.find("OK") != 0: + coinc_id = None + try: + f = open('/tmp/foo','a') + f.write("ERROR (stdin): %s\n" % out) + f.write("ERROR (stderr): %s\n" % err) + f.close() + except: pass + else: + coinc_id = out[2:].strip() + return coinc_id + def handle_uploaded_data(event, datafilename, log_filename='event.log', coinc_table_filename='coinc.xml'): @@ -65,8 +96,11 @@ def handle_uploaded_data(event, datafilename, event.instruments = coinc_table[0].instruments event.nevents = coinc_table[0].nevents event.likelihood = coinc_table[0].likelihood - event.save() + xml_filename = os.path.join(output_dir, coinc_table_filename) + event.coincEvent_id = insert_ligolw_tables(xml_filename) + + event.save() if event.analysisType == 'MBTA': #here's how it works for inspirals @@ -118,6 +152,10 @@ def handle_uploaded_data(event, datafilename, event.instruments = coinc_table[0].instruments event.nevents = coinc_table[0].nevents event.likelihood = coinc_table[0].likelihood + + xml_filename = os.path.join(output_dir, coinc_table_filename) + event.coinc_id = insert_ligolw_tables(xml_filename) + event.save() elif event.analysisType == 'OM': # Omega @@ -164,6 +202,10 @@ def handle_uploaded_data(event, datafilename, event.instruments = coinc_table[0].instruments event.nevents = coinc_table[0].nevents event.likelihood = coinc_table[0].likelihood + + xml_filename = os.path.join(output_dir, coinc_table_filename) + event.coinc_id = insert_ligolw_tables(xml_filename) + event.save() else: pass diff --git a/ligolw/__init__.py b/ligolw/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ligolw/insert.py b/ligolw/insert.py new file mode 100755 index 0000000000000000000000000000000000000000..a2e64694fcac71173646dd3631dd9ce3ef1039c7 --- /dev/null +++ b/ligolw/insert.py @@ -0,0 +1,140 @@ +#!/bin/env python + +# +# preamble +# + +import sys, os + +import StringIO + +import MySQLdb +from MySQLdb import OperationalError + +import glue +from glue.ligolw import ligolw +from glue.ligolw import lsctables +# FIXME: remove the next line when this attribute is initialized properly +# in lsctables.py +lsctables.SnglInspiralTable.next_id = lsctables.SnglInspiralID(0) +# FIXME: remove the next line when SnglInspiralTable no longer has its own +# custom version of this method +lsctables.SnglInspiralTable.updateKeyMapping = lsctables.table.Table.updateKeyMapping +from glue.ligolw import utils +from glue.ligolw.utils.ligolw_add import reassign_ids as ligolw_reassign_ids +# while importing dbtables, need before and after copies of the +# LIGOLWContentHandler class +class LIGOLWRAMContentHandler(ligolw.LIGOLWContentHandler): + startTable = ligolw.LIGOLWContentHandler.startTable + endTable = ligolw.LIGOLWContentHandler.endTable +from glue.ligolw import dbtables +class LIGOLWDBContentHandler(ligolw.LIGOLWContentHandler): + startTable = ligolw.LIGOLWContentHandler.startTable + endTable = ligolw.LIGOLWContentHandler.endTable + +dbtables.DBTable.maxrowid = lambda self: None + +# +# initialize the next_id attributes of all the table classes +# + +#------------------------------------------------------------------ + +#from insert_nextid import next_id +# included below. + +import MySQLdb +from MySQLdb.constants import ER + +TABLE_NAME = "ligolwids" + +create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME +select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME +insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME +update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME +lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME +unlock_sql = "UNLOCK TABLES" + +def create_id_table(connection): + connection.cursor().execute(create_table_sql) + +def db_get_next_id(connection, table_name, create_table=True, create_row=True): + cursor = connection.cursor() + try: + try: + cursor.execute(lock_sql) + cursor.execute(select_id_sql, [table_name]) + next = cursor.fetchone() + if next is None: + cursor.execute(insert_new_sql, [table_name]) + cursor.execute(select_id_sql, [table_name]) + next = cursor.fetchone() + cursor.execute(update_sql, [table_name]) + finally: + cursor.execute(unlock_sql) + cursor.close() + except MySQLdb.ProgrammingError, e: + if e.args[0] != ER.NO_SUCH_TABLE or not create_table: + raise + create_id_table(connection) + return db_get_next_id(connection, table_name, create_table=False) + return int(next[0]) + + +#------------------------------------------------------------------ + +def initialize(connection): + for cls in lsctables.TableByName.values(): + if cls.next_id is not None: + cls.get_next_id = lambda self: type(self.next_id)(db_get_next_id(connection, self.next_id.table_name)) + classmethod(cls.get_next_id) + +def insert_ligolw_tables(connection, filename, verbose=False): + + initialize(connection) + # + # parse .xml file into memory, reassign ids, write xml stream to in-ram + # buffer + # + xmldoc = ligolw_reassign_ids( + utils.load_filename( + filename, + gz = (filename or "stdin").endswith(".gz"), + verbose = verbose, + contenthandler = LIGOLWRAMContentHandler + ), + verbose = verbose + ) + + # <-- find coinc_event_id + coinc_table = glue.ligolw.table.get_table( + xmldoc, + glue.ligolw.lsctables.CoincTable.tableName) + coinc_id = coinc_table[0].coinc_event_id + + + buf = StringIO.StringIO() + utils.write_fileobj(xmldoc, buf) + buf.seek(0) + # + # re-parse xml stream from in-ram buffer to sqlite database + # + dbtables.DBTable_set_connection(connection) + xmldoc, digest = utils.load_fileobj(buf, contenthandler = LIGOLWDBContentHandler) + xmldoc.unlink() + dbtables.DBTable_set_connection(None) + + return coinc_id + +if __name__ == "__main__": + import sys + + user = sys.argv[1] + password = sys.argv[2] + db = sys.argv[3] + filename = sys.argv[4] + + conn= MySQLdb.connect(user=user, passwd=password, db=db) + + rv = insert_ligolw_tables(conn, filename) + print "OK", rv diff --git a/ligolw/insert_nextid.py b/ligolw/insert_nextid.py new file mode 100644 index 0000000000000000000000000000000000000000..9e6d78965bc9ecce139c7b59c43e9c3c5ebaf19a --- /dev/null +++ b/ligolw/insert_nextid.py @@ -0,0 +1,48 @@ +import MySQLdb +from MySQLdb.constants import ER + +conn = MySQLdb.connect(db='ligolw', user='root') + +TABLE_NAME = "ligolwids" + +create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME +select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME +insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME +update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME +lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME +unlock_sql = "UNLOCK TABLES" + +def create_id_table(connection): + connection.cursor().execute(create_table_sql) + +def next_id(connection, table_name, create_table=True, create_row=True): + next = None + cursor = connection.cursor() + try: + try: + cursor.execute(lock_sql) + cursor.execute(select_id_sql, [table_name]) + next = cursor.fetchone() + if not next: + cursor.execute(insert_new_sql, [table_name]) + cursor.execute(select_id_sql, [table_name]) + next = cursor.fetchone() + cursor.execute(update_sql, [table_name]) + finally: + cursor.execute(unlock_sql) + cursor.close() + except MySQLdb.ProgrammingError, e: + # XXX remove + global foo + foo = e + cursor.close() + if e.args[0] == ER.NO_SUCH_TABLE: + if not create_table: + raise + create_id_table(connection) + return next_id(connection, table_name, create_table=False) + else: + raise + cursor.close() + return next[0] + diff --git a/ligolw/models.py b/ligolw/models.py new file mode 100644 index 0000000000000000000000000000000000000000..1b36b986bf4008365b88d5c06552bc2307260f4b --- /dev/null +++ b/ligolw/models.py @@ -0,0 +1,228 @@ + +# This is an auto-generated Django model module. +# You'll have to do the following manually to clean this up: +# * Rearrange models' order +# * Make sure each model has one field with primary_key=True +# Feel free to rename the models, but don't rename db_table values or field names. +# +# Also note: +# You'll have to insert the output of 'django-admin.py sqlcustom [appname]' +# into your database. + +from django.db import models + +class CoincDefiner(models.Model): + search = models.CharField(max_length=150, blank=True) + description = models.CharField(max_length=150, blank=True) + coinc_def_id = models.CharField(max_length=150, primary_key=True) + search_coinc_type = models.IntegerField(null=True, blank=True) + class Meta: + db_table = u'coinc_definer' + +class TimeSlide(models.Model): + instrument = models.CharField(max_length=150, primary_key=True) + time_slide_id = models.CharField(max_length=150, primary_key=True) + process_id = models.CharField(max_length=150, blank=True) + offset = models.FloatField(null=True, blank=True) + class Meta: + db_table = u'time_slide' + +class Process(models.Model): + program = models.CharField(max_length=150, blank=True) + version = models.CharField(max_length=150, blank=True) + cvs_repository = models.CharField(max_length=150, blank=True) + cvs_entry_time = models.IntegerField(null=True, blank=True) + comment = models.CharField(max_length=150, blank=True) + is_online = models.IntegerField(null=True, blank=True) + node = models.CharField(max_length=150, blank=True) + username = models.CharField(max_length=150, blank=True) + unix_procid = models.IntegerField(null=True, blank=True) + start_time = models.IntegerField(null=True, blank=True) + end_time = models.IntegerField(null=True, blank=True) + jobid = models.IntegerField(null=True, blank=True) + domain = models.CharField(max_length=150, blank=True) + ifos = models.CharField(max_length=150, blank=True) + process_id = models.CharField(max_length=150, primary_key=True) + class Meta: + db_table = u'process' + + +class CoincEvent(models.Model): + coinc_event_id = models.CharField(max_length=150, primary_key=True) + instruments = models.CharField(max_length=150, blank=True) + nevents = models.IntegerField(null=True, blank=True) + #process_id = models.CharField(max_length=150, blank=True) + process = models.ForeignKey(Process, blank=True) + #coinc_def_id = models.CharField(max_length=150, blank=True) + coinc_def = models.ForeignKey(CoincDefiner, blank=True) + #time_slide_id = models.CharField(max_length=150, blank=True) + time_slide = models.ForeignKey(TimeSlide, blank=True) + likelihood = models.FloatField(null=True, blank=True) + class Meta: + db_table = u'coinc_event' + +class CoincEventMap(models.Model): + event_id = models.CharField(max_length=150, blank=True) + table_name = models.CharField(max_length=150, blank=True) + coinc_event_id = models.CharField(max_length=150, blank=True) + class Meta: + db_table = u'coinc_event_map' + +class CoincInspiral(models.Model): + false_alarm_rate = models.FloatField(null=True, blank=True) + mchirp = models.FloatField(null=True, blank=True) + #coinc_event_id = models.CharField(max_length=150, blank=True) + coinc_event = models.OneToOneField(CoincEvent, primary_key=True) + combined_far = models.FloatField(null=True, blank=True) + mass = models.FloatField(null=True, blank=True) + end_time = models.IntegerField(null=True, blank=True) + snr = models.FloatField(null=True, blank=True) + end_time_ns = models.IntegerField(null=True, blank=True) + ifos = models.CharField(max_length=150, blank=True) + class Meta: + db_table = u'coinc_inspiral' + + +class Experiment(models.Model): + search = models.CharField(max_length=150, blank=True) + instruments = models.CharField(max_length=150, blank=True) + search_group = models.CharField(max_length=150, blank=True) + comments = models.CharField(max_length=150, blank=True) + lars_id = models.CharField(max_length=150, blank=True) + experiment_id = models.CharField(max_length=150, primary_key=True) + gps_start_time = models.IntegerField(null=True, blank=True) + gps_end_time = models.IntegerField(null=True, blank=True) + class Meta: + db_table = u'experiment' + +class ExperimentMap(models.Model): + experiment_summ_id = models.CharField(max_length=150, blank=True) + #coinc_event_id = models.CharField(max_length=150, blank=True) + coinc_event = models.ForeignKey(CoincEvent, blank=True) + class Meta: + db_table = u'experiment_map' + +class ExperimentSummary(models.Model): + sim_proc_id = models.CharField(max_length=150, blank=True) + datatype = models.CharField(max_length=150, blank=True) + experiment_summ_id = models.CharField(max_length=150, primary_key=True) + nevents = models.IntegerField(null=True, blank=True) + #experiment_id = models.CharField(max_length=150, blank=True) + experiment = models.ForeignKey(Experiment, blank=True) + duration = models.IntegerField(null=True, blank=True) + #time_slide_id = models.CharField(max_length=150, blank=True) + time_slide = models.ForeignKey(TimeSlide, blank=True) + veto_def_name = models.CharField(max_length=150, blank=True) + rowid = models.IntegerField(unique=True, db_column='ROWID') # Field name made lowercase. + class Meta: + db_table = u'experiment_summary' + +class Ligolwids(models.Model): + tablename = models.CharField(max_length=90, primary_key=True) + nextid = models.IntegerField(null=True, blank=True) + class Meta: + db_table = u'ligolwids' + +class ProcessParams(models.Model): + program = models.CharField(max_length=150, blank=True) + #process_id = models.CharField(max_length=150, blank=True) + process = models.ForeignKey(Process, blank=True) + param = models.CharField(max_length=150, blank=True) + type = models.CharField(max_length=150, blank=True) + value = models.CharField(max_length=150, blank=True) + class Meta: + db_table = u'process_params' + +class SearchSummary(models.Model): + #process_id = models.CharField(max_length=150, blank=True) + process = models.ForeignKey(Process, blank=True) + shared_object = models.CharField(max_length=150, blank=True) + lalwrapper_cvs_tag = models.CharField(max_length=150, blank=True) + lal_cvs_tag = models.CharField(max_length=150, blank=True) + comment = models.CharField(max_length=150, blank=True) + ifos = models.CharField(max_length=150, blank=True) + in_start_time = models.IntegerField(null=True, blank=True) + in_start_time_ns = models.IntegerField(null=True, blank=True) + in_end_time = models.IntegerField(null=True, blank=True) + in_end_time_ns = models.IntegerField(null=True, blank=True) + out_start_time = models.IntegerField(null=True, blank=True) + out_start_time_ns = models.IntegerField(null=True, blank=True) + out_end_time = models.IntegerField(null=True, blank=True) + out_end_time_ns = models.IntegerField(null=True, blank=True) + nevents = models.IntegerField(null=True, blank=True) + nnodes = models.IntegerField(null=True, blank=True) + class Meta: + db_table = u'search_summary' + +class SearchSummvars(models.Model): + #process_id = models.CharField(max_length=150, blank=True) + process = models.ForeignKey(Process, blank=True) + name = models.CharField(max_length=150, blank=True) + string = models.CharField(max_length=150, blank=True) + value = models.FloatField(null=True, blank=True) + search_summvar_id = models.CharField(max_length=150, primary_key=True) + class Meta: + db_table = u'search_summvars' + +class SnglInspiral(models.Model): + process_id = models.CharField(max_length=150, blank=True) + ifo = models.CharField(max_length=150, blank=True) + search = models.CharField(max_length=150, blank=True) + channel = models.CharField(max_length=150, blank=True) + end_time = models.IntegerField(null=True, blank=True) + end_time_ns = models.IntegerField(null=True, blank=True) + end_time_gmst = models.FloatField(null=True, blank=True) + impulse_time = models.IntegerField(null=True, blank=True) + impulse_time_ns = models.IntegerField(null=True, blank=True) + template_duration = models.FloatField(null=True, blank=True) + event_duration = models.FloatField(null=True, blank=True) + amplitude = models.FloatField(null=True, blank=True) + eff_distance = models.FloatField(null=True, blank=True) + coa_phase = models.FloatField(null=True, blank=True) + mass1 = models.FloatField(null=True, blank=True) + mass2 = models.FloatField(null=True, blank=True) + mchirp = models.FloatField(null=True, blank=True) + mtotal = models.FloatField(null=True, blank=True) + eta = models.FloatField(null=True, blank=True) + kappa = models.FloatField(null=True, blank=True) + chi = models.FloatField(null=True, blank=True) + tau0 = models.FloatField(null=True, blank=True) + tau2 = models.FloatField(null=True, blank=True) + tau3 = models.FloatField(null=True, blank=True) + tau4 = models.FloatField(null=True, blank=True) + tau5 = models.FloatField(null=True, blank=True) + ttotal = models.FloatField(null=True, blank=True) + psi0 = models.FloatField(null=True, blank=True) + psi3 = models.FloatField(null=True, blank=True) + alpha = models.FloatField(null=True, blank=True) + alpha1 = models.FloatField(null=True, blank=True) + alpha2 = models.FloatField(null=True, blank=True) + alpha3 = models.FloatField(null=True, blank=True) + alpha4 = models.FloatField(null=True, blank=True) + alpha5 = models.FloatField(null=True, blank=True) + alpha6 = models.FloatField(null=True, blank=True) + beta = models.FloatField(null=True, blank=True) + f_final = models.FloatField(null=True, blank=True) + snr = models.FloatField(null=True, blank=True) + chisq = models.FloatField(null=True, blank=True) + chisq_dof = models.IntegerField(null=True, blank=True) + bank_chisq = models.FloatField(null=True, blank=True) + bank_chisq_dof = models.IntegerField(null=True, blank=True) + cont_chisq = models.FloatField(null=True, blank=True) + cont_chisq_dof = models.IntegerField(null=True, blank=True) + sigmasq = models.FloatField(null=True, blank=True) + rsqveto_duration = models.FloatField(null=True, blank=True) + gamma0 = models.FloatField(null=True, db_column='Gamma0', blank=True) # Field name made lowercase. + gamma1 = models.FloatField(null=True, db_column='Gamma1', blank=True) # Field name made lowercase. + gamma2 = models.FloatField(null=True, db_column='Gamma2', blank=True) # Field name made lowercase. + gamma3 = models.FloatField(null=True, db_column='Gamma3', blank=True) # Field name made lowercase. + gamma4 = models.FloatField(null=True, db_column='Gamma4', blank=True) # Field name made lowercase. + gamma5 = models.FloatField(null=True, db_column='Gamma5', blank=True) # Field name made lowercase. + gamma6 = models.FloatField(null=True, db_column='Gamma6', blank=True) # Field name made lowercase. + gamma7 = models.FloatField(null=True, db_column='Gamma7', blank=True) # Field name made lowercase. + gamma8 = models.FloatField(null=True, db_column='Gamma8', blank=True) # Field name made lowercase. + gamma9 = models.FloatField(null=True, db_column='Gamma9', blank=True) # Field name made lowercase. + event_id = models.CharField(max_length=150, primary_key=True) + class Meta: + db_table = u'sngl_inspiral' +