Added ligolw database population code.

REQUIRES db schema mods in upgrade-3-4.sql

gracedb/models.py

@@ -4,6 +4,8 @@ import thread
 import string
 import os
 
+from gracedb.ligolw.models import CoincEvent
+
 class User(models.Model):
     name = models.CharField(max_length=100)
     email = models.EmailField()
@@ -45,6 +47,7 @@ class Event(models.Model):
     instruments = models.CharField(max_length=20, default="")
     nevents = models.PositiveIntegerField(null=True)
     likelihood = models.FloatField(null=True)
+    coincEvent = models.ForeignKey(CoincEvent, null=True)
 
     # NOT from coinc_event, but so, so common.
     #   Note that the semantics for this is different depending

gracedb/translator.py

 
-import os
+import os, sys
 
 from models import EventLog
+from subprocess import Popen, PIPE
 
 import glue, glue.ligolw.utils
 from glue.gracedb.utils import InspiralCoincDef
@@ -15,6 +16,36 @@ from glue.gracedb.utils import populate_inspiral_tables, \
                                populate_coinc_tables,    \
                                write_output_files
 
+# Importing this messes with other ligolw table actions.
+#from gracedb.ligolw.insert import insert_ligolw_tables
+import gracedb.ligolw
+
+def insert_ligolw_tables(xml_filename):
+    #insert_ligolw_tables(django.db.connection, xml_filename)
+    prog = os.path.dirname(gracedb.ligolw.__file__)
+    prog = os.path.join(prog, "insert.py")
+    e = dict(os.environ)
+    ppath = e.get("PYTHONPATH") or ""
+    ppath = ppath.split(':')
+    ppath = ppath + sys.path
+    e['PYTHONPATH'] = ':'.join(ppath)
+    p = Popen( (prog, "root", "", "gracedb", xml_filename), stdout=PIPE, stderr=PIPE, env=e)
+    out = p.stdout.read()
+    err = p.stderr.read()
+    p.wait()
+    out += p.stdout.read()
+    if out.find("OK") != 0:
+        coinc_id = None
+        try:
+            f = open('/tmp/foo','a')
+            f.write("ERROR (stdin): %s\n" % out)
+            f.write("ERROR (stderr): %s\n" % err)
+            f.close()
+        except: pass
+    else:
+        coinc_id = out[2:].strip()
+    return coinc_id
+
 def handle_uploaded_data(event, datafilename,
                          log_filename='event.log',
                          coinc_table_filename='coinc.xml'):
@@ -64,8 +95,11 @@ def handle_uploaded_data(event, datafilename,
         event.instruments = coinc_table[0].instruments
         event.nevents = coinc_table[0].nevents
         event.likelihood = coinc_table[0].likelihood
-        event.save()
 
+        xml_filename = os.path.join(output_dir, coinc_table_filename)
+        event.coincEvent_id = insert_ligolw_tables(xml_filename)
+
+        event.save()
 
     if event.analysisType == 'MBTA':
         #xmldoc, log_data, detectors, cid = populate_inspiral_tables("MbtaFake-930909680-16.gwf")
@@ -121,6 +155,10 @@ def handle_uploaded_data(event, datafilename,
         event.instruments = coinc_table[0].instruments
         event.nevents = coinc_table[0].nevents
         event.likelihood = coinc_table[0].likelihood
+
+        xml_filename = os.path.join(output_dir, coinc_table_filename)
+        event.coinc_id = insert_ligolw_tables(xml_filename)
+
         event.save()
 
     elif event.analysisType == 'OM': # Omega
@@ -172,6 +210,10 @@ def handle_uploaded_data(event, datafilename,
         event.instruments = coinc_table[0].instruments
         event.nevents = coinc_table[0].nevents
         event.likelihood = coinc_table[0].likelihood
+
+        xml_filename = os.path.join(output_dir, coinc_table_filename)
+        event.coinc_id = insert_ligolw_tables(xml_filename)
+
         event.save()
     else:
         pass

ligolw/insert.py

+#!/bin/env python
+
+#
+# preamble
+#
+
+import sys, os
+
+import StringIO
+
+import MySQLdb
+from MySQLdb import OperationalError
+
+import glue
+from glue.ligolw import ligolw
+from glue.ligolw import lsctables
+# FIXME:  remove the next line when this attribute is initialized properly
+# in lsctables.py
+lsctables.SnglInspiralTable.next_id = lsctables.SnglInspiralID(0)
+# FIXME:  remove the next line when SnglInspiralTable no longer has its own
+# custom version of this method
+lsctables.SnglInspiralTable.updateKeyMapping = lsctables.table.Table.updateKeyMapping
+from glue.ligolw import utils
+from glue.ligolw.utils.ligolw_add import reassign_ids as ligolw_reassign_ids
+# while importing dbtables, need before and after copies of the
+# LIGOLWContentHandler class
+class LIGOLWRAMContentHandler(ligolw.LIGOLWContentHandler):
+    startTable = ligolw.LIGOLWContentHandler.startTable
+    endTable = ligolw.LIGOLWContentHandler.endTable
+from glue.ligolw import dbtables
+class LIGOLWDBContentHandler(ligolw.LIGOLWContentHandler):
+    startTable = ligolw.LIGOLWContentHandler.startTable
+    endTable = ligolw.LIGOLWContentHandler.endTable
+
+dbtables.DBTable.maxrowid = lambda self: None
+
+#
+# initialize the next_id attributes of all the table classes
+#
+
+#------------------------------------------------------------------
+
+#from insert_nextid import next_id
+# included below.
+
+import MySQLdb
+from MySQLdb.constants import ER
+
+TABLE_NAME = "ligolwids"
+
+create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME
+select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME
+insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME
+update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME
+lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME
+unlock_sql = "UNLOCK TABLES"
+
+def create_id_table(connection):
+    connection.cursor().execute(create_table_sql)
+
+def db_get_next_id(connection, table_name, create_table=True, create_row=True):
+    cursor = connection.cursor()
+    try:
+        try:
+            cursor.execute(lock_sql)
+            cursor.execute(select_id_sql, [table_name])
+            next = cursor.fetchone()
+            if next is None:
+                cursor.execute(insert_new_sql, [table_name])
+                cursor.execute(select_id_sql, [table_name])
+                next = cursor.fetchone()
+            cursor.execute(update_sql, [table_name])
+        finally:
+            cursor.execute(unlock_sql)
+            cursor.close()
+    except MySQLdb.ProgrammingError, e:
+        if e.args[0] != ER.NO_SUCH_TABLE or not create_table:
+            raise
+        create_id_table(connection)
+        return db_get_next_id(connection, table_name, create_table=False)
+    return int(next[0])
+
+
+#------------------------------------------------------------------
+
+def initialize(connection):
+    for cls in lsctables.TableByName.values():
+        if cls.next_id is not None:
+            cls.get_next_id = lambda self: type(self.next_id)(db_get_next_id(connection, self.next_id.table_name))
+            classmethod(cls.get_next_id)
+
+def insert_ligolw_tables(connection, filename, verbose=False):
+
+    initialize(connection)
+    #
+    # parse .xml file into memory, reassign ids, write xml stream to in-ram
+    # buffer
+    #
+    xmldoc = ligolw_reassign_ids(
+        utils.load_filename(
+            filename,
+            gz = (filename or "stdin").endswith(".gz"),
+            verbose = verbose,
+            contenthandler = LIGOLWRAMContentHandler
+        ),
+        verbose = verbose
+    )
+
+    # <-- find coinc_event_id
+    coinc_table = glue.ligolw.table.get_table(
+            xmldoc,
+            glue.ligolw.lsctables.CoincTable.tableName)
+    coinc_id = coinc_table[0].coinc_event_id
+
+
+    buf = StringIO.StringIO()
+    utils.write_fileobj(xmldoc, buf)
+    buf.seek(0)
+    #
+    # re-parse xml stream from in-ram buffer to sqlite database
+    #
+    dbtables.DBTable_set_connection(connection)
+    xmldoc, digest = utils.load_fileobj(buf, contenthandler = LIGOLWDBContentHandler)
+    xmldoc.unlink()
+    dbtables.DBTable_set_connection(None)
+
+    return coinc_id
+
+if __name__ == "__main__":
+    import sys
+
+    user = sys.argv[1]
+    password = sys.argv[2]
+    db = sys.argv[3]
+    filename = sys.argv[4]
+
+    conn= MySQLdb.connect(user=user, passwd=password, db=db)
+
+    rv = insert_ligolw_tables(conn, filename)
+    print "OK", rv

ligolw/insert_nextid.py

+import MySQLdb
+from MySQLdb.constants import ER
+
+conn = MySQLdb.connect(db='ligolw', user='root')
+
+TABLE_NAME = "ligolwids"
+
+create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME
+select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME
+insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME
+update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME
+lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME
+unlock_sql = "UNLOCK TABLES"
+
+def create_id_table(connection):
+    connection.cursor().execute(create_table_sql)
+
+def next_id(connection, table_name, create_table=True, create_row=True):
+    next = None
+    cursor = connection.cursor()
+    try:
+        try:
+            cursor.execute(lock_sql)
+            cursor.execute(select_id_sql, [table_name])
+            next = cursor.fetchone()
+            if not next:
+                cursor.execute(insert_new_sql, [table_name])
+                cursor.execute(select_id_sql, [table_name])
+                next = cursor.fetchone()
+            cursor.execute(update_sql, [table_name])
+        finally:
+            cursor.execute(unlock_sql)
+            cursor.close()
+    except MySQLdb.ProgrammingError, e:
+        # XXX remove
+        global foo
+        foo = e
+        cursor.close()
+        if e.args[0] == ER.NO_SUCH_TABLE:
+            if not create_table:
+                raise
+            create_id_table(connection)
+            return next_id(connection, table_name, create_table=False)
+        else:
+            raise
+    cursor.close()
+    return next[0]
+

ligolw/models.py

+
+# This is an auto-generated Django model module.
+# You'll have to do the following manually to clean this up:
+#     * Rearrange models' order
+#     * Make sure each model has one field with primary_key=True
+# Feel free to rename the models, but don't rename db_table values or field names.
+#
+# Also note:
+#   You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
+#   into your database.
+
+from django.db import models
+
+class CoincDefiner(models.Model):
+    search = models.CharField(max_length=150, blank=True)
+    description = models.CharField(max_length=150, blank=True)
+    coinc_def_id = models.CharField(max_length=150, primary_key=True)
+    search_coinc_type = models.IntegerField(null=True, blank=True)
+    class Meta:
+        db_table = u'coinc_definer'
+
+class TimeSlide(models.Model):
+    instrument = models.CharField(max_length=150, primary_key=True)
+    time_slide_id = models.CharField(max_length=150, primary_key=True)
+    process_id = models.CharField(max_length=150, blank=True)
+    offset = models.FloatField(null=True, blank=True)
+    class Meta:
+        db_table = u'time_slide'
+
+class Process(models.Model):
+    program = models.CharField(max_length=150, blank=True)
+    version = models.CharField(max_length=150, blank=True)
+    cvs_repository = models.CharField(max_length=150, blank=True)
+    cvs_entry_time = models.IntegerField(null=True, blank=True)
+    comment = models.CharField(max_length=150, blank=True)
+    is_online = models.IntegerField(null=True, blank=True)
+    node = models.CharField(max_length=150, blank=True)
+    username = models.CharField(max_length=150, blank=True)
+    unix_procid = models.IntegerField(null=True, blank=True)
+    start_time = models.IntegerField(null=True, blank=True)
+    end_time = models.IntegerField(null=True, blank=True)
+    jobid = models.IntegerField(null=True, blank=True)
+    domain = models.CharField(max_length=150, blank=True)
+    ifos = models.CharField(max_length=150, blank=True)
+    process_id = models.CharField(max_length=150, primary_key=True)
+    class Meta:
+        db_table = u'process'
+
+
+class CoincEvent(models.Model):
+    coinc_event_id = models.CharField(max_length=150, primary_key=True)
+    instruments = models.CharField(max_length=150, blank=True)
+    nevents = models.IntegerField(null=True, blank=True)
+    #process_id = models.CharField(max_length=150, blank=True)
+    process =  models.ForeignKey(Process, blank=True)
+    #coinc_def_id = models.CharField(max_length=150, blank=True)
+    coinc_def = models.ForeignKey(CoincDefiner, blank=True)
+    #time_slide_id = models.CharField(max_length=150, blank=True)
+    time_slide = models.ForeignKey(TimeSlide, blank=True)
+    likelihood = models.FloatField(null=True, blank=True)
+    class Meta:
+        db_table = u'coinc_event'
+
+class CoincEventMap(models.Model):
+    event_id = models.CharField(max_length=150, blank=True)
+    table_name = models.CharField(max_length=150, blank=True)
+    coinc_event_id = models.CharField(max_length=150, blank=True)
+    class Meta:
+        db_table = u'coinc_event_map'
+
+class CoincInspiral(models.Model):
+    false_alarm_rate = models.FloatField(null=True, blank=True)
+    mchirp = models.FloatField(null=True, blank=True)
+    #coinc_event_id = models.CharField(max_length=150, blank=True)
+    coinc_event = models.OneToOneField(CoincEvent, primary_key=True)
+    combined_far = models.FloatField(null=True, blank=True)
+    mass = models.FloatField(null=True, blank=True)
+    end_time = models.IntegerField(null=True, blank=True)
+    snr = models.FloatField(null=True, blank=True)
+    end_time_ns = models.IntegerField(null=True, blank=True)
+    ifos = models.CharField(max_length=150, blank=True)
+    class Meta:
+        db_table = u'coinc_inspiral'
+
+
+class Experiment(models.Model):
+    search = models.CharField(max_length=150, blank=True)
+    instruments = models.CharField(max_length=150, blank=True)
+    search_group = models.CharField(max_length=150, blank=True)
+    comments = models.CharField(max_length=150, blank=True)
+    lars_id = models.CharField(max_length=150, blank=True)
+    experiment_id = models.CharField(max_length=150, primary_key=True)
+    gps_start_time = models.IntegerField(null=True, blank=True)
+    gps_end_time = models.IntegerField(null=True, blank=True)
+    class Meta:
+        db_table = u'experiment'
+
+class ExperimentMap(models.Model):
+    experiment_summ_id = models.CharField(max_length=150, blank=True)
+    #coinc_event_id = models.CharField(max_length=150, blank=True)
+    coinc_event = models.ForeignKey(CoincEvent, blank=True)
+    class Meta:
+        db_table = u'experiment_map'
+
+class ExperimentSummary(models.Model):
+    sim_proc_id = models.CharField(max_length=150, blank=True)
+    datatype = models.CharField(max_length=150, blank=True)
+    experiment_summ_id = models.CharField(max_length=150, primary_key=True)
+    nevents = models.IntegerField(null=True, blank=True)
+    #experiment_id = models.CharField(max_length=150, blank=True)
+    experiment = models.ForeignKey(Experiment, blank=True)
+    duration = models.IntegerField(null=True, blank=True)
+    #time_slide_id = models.CharField(max_length=150, blank=True)
+    time_slide = models.ForeignKey(TimeSlide, blank=True)
+    veto_def_name = models.CharField(max_length=150, blank=True)
+    rowid = models.IntegerField(unique=True, db_column='ROWID') # Field name made lowercase.
+    class Meta:
+        db_table = u'experiment_summary'
+
+class Ligolwids(models.Model):
+    tablename = models.CharField(max_length=90, primary_key=True)
+    nextid = models.IntegerField(null=True, blank=True)
+    class Meta:
+        db_table = u'ligolwids'
+
+class ProcessParams(models.Model):
+    program = models.CharField(max_length=150, blank=True)
+    #process_id = models.CharField(max_length=150, blank=True)
+    process = models.ForeignKey(Process, blank=True)
+    param = models.CharField(max_length=150, blank=True)
+    type = models.CharField(max_length=150, blank=True)
+    value = models.CharField(max_length=150, blank=True)
+    class Meta:
+        db_table = u'process_params'
+
+class SearchSummary(models.Model):
+    #process_id = models.CharField(max_length=150, blank=True)
+    process = models.ForeignKey(Process, blank=True)
+    shared_object = models.CharField(max_length=150, blank=True)
+    lalwrapper_cvs_tag = models.CharField(max_length=150, blank=True)
+    lal_cvs_tag = models.CharField(max_length=150, blank=True)
+    comment = models.CharField(max_length=150, blank=True)
+    ifos = models.CharField(max_length=150, blank=True)
+    in_start_time = models.IntegerField(null=True, blank=True)
+    in_start_time_ns = models.IntegerField(null=True, blank=True)
+    in_end_time = models.IntegerField(null=True, blank=True)
+    in_end_time_ns = models.IntegerField(null=True, blank=True)
+    out_start_time = models.IntegerField(null=True, blank=True)
+    out_start_time_ns = models.IntegerField(null=True, blank=True)
+    out_end_time = models.IntegerField(null=True, blank=True)
+    out_end_time_ns = models.IntegerField(null=True, blank=True)
+    nevents = models.IntegerField(null=True, blank=True)
+    nnodes = models.IntegerField(null=True, blank=True)
+    class Meta:
+        db_table = u'search_summary'
+
+class SearchSummvars(models.Model):
+    #process_id = models.CharField(max_length=150, blank=True)
+    process = models.ForeignKey(Process, blank=True)
+    name = models.CharField(max_length=150, blank=True)
+    string = models.CharField(max_length=150, blank=True)
+    value = models.FloatField(null=True, blank=True)
+    search_summvar_id = models.CharField(max_length=150, primary_key=True)
+    class Meta:
+        db_table = u'search_summvars'
+
+class SnglInspiral(models.Model):
+    process_id = models.CharField(max_length=150, blank=True)
+    ifo = models.CharField(max_length=150, blank=True)
+    search = models.CharField(max_length=150, blank=True)
+    channel = models.CharField(max_length=150, blank=True)
+    end_time = models.IntegerField(null=True, blank=True)
+    end_time_ns = models.IntegerField(null=True, blank=True)
+    end_time_gmst = models.FloatField(null=True, blank=True)
+    impulse_time = models.IntegerField(null=True, blank=True)
+    impulse_time_ns = models.IntegerField(null=True, blank=True)
+    template_duration = models.FloatField(null=True, blank=True)
+    event_duration = models.FloatField(null=True, blank=True)
+    amplitude = models.FloatField(null=True, blank=True)
+    eff_distance = models.FloatField(null=True, blank=True)
+    coa_phase = models.FloatField(null=True, blank=True)
+    mass1 = models.FloatField(null=True, blank=True)
+    mass2 = models.FloatField(null=True, blank=True)
+    mchirp = models.FloatField(null=True, blank=True)
+    mtotal = models.FloatField(null=True, blank=True)
+    eta = models.FloatField(null=True, blank=True)
+    kappa = models.FloatField(null=True, blank=True)
+    chi = models.FloatField(null=True, blank=True)
+    tau0 = models.FloatField(null=True, blank=True)
+    tau2 = models.FloatField(null=True, blank=True)
+    tau3 = models.FloatField(null=True, blank=True)
+    tau4 = models.FloatField(null=True, blank=True)
+    tau5 = models.FloatField(null=True, blank=True)
+    ttotal = models.FloatField(null=True, blank=True)
+    psi0 = models.FloatField(null=True, blank=True)
+    psi3 = models.FloatField(null=True, blank=True)
+    alpha = models.FloatField(null=True, blank=True)
+    alpha1 = models.FloatField(null=True, blank=True)
+    alpha2 = models.FloatField(null=True, blank=True)
+    alpha3 = models.FloatField(null=True, blank=True)
+    alpha4 = models.FloatField(null=True, blank=True)
+    alpha5 = models.FloatField(null=True, blank=True)
+    alpha6 = models.FloatField(null=True, blank=True)
+    beta = models.FloatField(null=True, blank=True)
+    f_final = models.FloatField(null=True, blank=True)
+    snr = models.FloatField(null=True, blank=True)
+    chisq = models.FloatField(null=True, blank=True)
+    chisq_dof = models.IntegerField(null=True, blank=True)
+    bank_chisq = models.FloatField(null=True, blank=True)
+    bank_chisq_dof = models.IntegerField(null=True, blank=True)
+    cont_chisq = models.FloatField(null=True, blank=True)
+    cont_chisq_dof = models.IntegerField(null=True, blank=True)
+    sigmasq = models.FloatField(null=True, blank=True)
+    rsqveto_duration = models.FloatField(null=True, blank=True)
+    gamma0 = models.FloatField(null=True, db_column='Gamma0', blank=True) # Field name made lowercase.
+    gamma1 = models.FloatField(null=True, db_column='Gamma1', blank=True) # Field name made lowercase.
+    gamma2 = models.FloatField(null=True, db_column='Gamma2', blank=True) # Field name made lowercase.
+    gamma3 = models.FloatField(null=True, db_column='Gamma3', blank=True) # Field name made lowercase.
+    gamma4 = models.FloatField(null=True, db_column='Gamma4', blank=True) # Field name made lowercase.
+    gamma5 = models.FloatField(null=True, db_column='Gamma5', blank=True) # Field name made lowercase.
+    gamma6 = models.FloatField(null=True, db_column='Gamma6', blank=True) # Field name made lowercase.
+    gamma7 = models.FloatField(null=True, db_column='Gamma7', blank=True) # Field name made lowercase.
+    gamma8 = models.FloatField(null=True, db_column='Gamma8', blank=True) # Field name made lowercase.
+    gamma9 = models.FloatField(null=True, db_column='Gamma9', blank=True) # Field name made lowercase.
+    event_id = models.CharField(max_length=150, primary_key=True)
+    class Meta:
+        db_table = u'sngl_inspiral'
+

settings_dev.py

@@ -104,4 +104,5 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.sites',
     'gracedb.gracedb',
+    #'gracedb.ligolw',
 )