Skip to content
Snippets Groups Projects
Commit 71e57f40 authored by Brian Moe's avatar Brian Moe
Browse files

Added ligolw database population code.

REQUIRES db schema mods in upgrade-3-4.sql
parent 5ad7a3df
No related branches found
No related tags found
No related merge requests found
......@@ -4,6 +4,8 @@ import thread
import string
import os
from gracedb.ligolw.models import CoincEvent
class User(models.Model):
name = models.CharField(max_length=100)
email = models.EmailField()
......@@ -45,6 +47,7 @@ class Event(models.Model):
instruments = models.CharField(max_length=20, default="")
nevents = models.PositiveIntegerField(null=True)
likelihood = models.FloatField(null=True)
coincEvent = models.ForeignKey(CoincEvent, null=True)
# NOT from coinc_event, but so, so common.
# Note that the semantics for this is different depending
......
import os
import os, sys
from models import EventLog
from subprocess import Popen, PIPE
import glue, glue.ligolw.utils
from glue.gracedb.utils import InspiralCoincDef
......@@ -15,6 +16,36 @@ from glue.gracedb.utils import populate_inspiral_tables, \
populate_coinc_tables, \
write_output_files
# Importing this messes with other ligolw table actions.
#from gracedb.ligolw.insert import insert_ligolw_tables
import gracedb.ligolw
def insert_ligolw_tables(xml_filename):
#insert_ligolw_tables(django.db.connection, xml_filename)
prog = os.path.dirname(gracedb.ligolw.__file__)
prog = os.path.join(prog, "insert.py")
e = dict(os.environ)
ppath = e.get("PYTHONPATH") or ""
ppath = ppath.split(':')
ppath = ppath + sys.path
e['PYTHONPATH'] = ':'.join(ppath)
p = Popen( (prog, "root", "", "gracedb", xml_filename), stdout=PIPE, stderr=PIPE, env=e)
out = p.stdout.read()
err = p.stderr.read()
p.wait()
out += p.stdout.read()
if out.find("OK") != 0:
coinc_id = None
try:
f = open('/tmp/foo','a')
f.write("ERROR (stdin): %s\n" % out)
f.write("ERROR (stderr): %s\n" % err)
f.close()
except: pass
else:
coinc_id = out[2:].strip()
return coinc_id
def handle_uploaded_data(event, datafilename,
log_filename='event.log',
coinc_table_filename='coinc.xml'):
......@@ -64,8 +95,11 @@ def handle_uploaded_data(event, datafilename,
event.instruments = coinc_table[0].instruments
event.nevents = coinc_table[0].nevents
event.likelihood = coinc_table[0].likelihood
event.save()
xml_filename = os.path.join(output_dir, coinc_table_filename)
event.coincEvent_id = insert_ligolw_tables(xml_filename)
event.save()
if event.analysisType == 'MBTA':
#xmldoc, log_data, detectors, cid = populate_inspiral_tables("MbtaFake-930909680-16.gwf")
......@@ -121,6 +155,10 @@ def handle_uploaded_data(event, datafilename,
event.instruments = coinc_table[0].instruments
event.nevents = coinc_table[0].nevents
event.likelihood = coinc_table[0].likelihood
xml_filename = os.path.join(output_dir, coinc_table_filename)
event.coinc_id = insert_ligolw_tables(xml_filename)
event.save()
elif event.analysisType == 'OM': # Omega
......@@ -172,6 +210,10 @@ def handle_uploaded_data(event, datafilename,
event.instruments = coinc_table[0].instruments
event.nevents = coinc_table[0].nevents
event.likelihood = coinc_table[0].likelihood
xml_filename = os.path.join(output_dir, coinc_table_filename)
event.coinc_id = insert_ligolw_tables(xml_filename)
event.save()
else:
pass
#!/bin/env python
#
# preamble
#
import sys, os
import StringIO
import MySQLdb
from MySQLdb import OperationalError
import glue
from glue.ligolw import ligolw
from glue.ligolw import lsctables
# FIXME: remove the next line when this attribute is initialized properly
# in lsctables.py
lsctables.SnglInspiralTable.next_id = lsctables.SnglInspiralID(0)
# FIXME: remove the next line when SnglInspiralTable no longer has its own
# custom version of this method
lsctables.SnglInspiralTable.updateKeyMapping = lsctables.table.Table.updateKeyMapping
from glue.ligolw import utils
from glue.ligolw.utils.ligolw_add import reassign_ids as ligolw_reassign_ids
# while importing dbtables, need before and after copies of the
# LIGOLWContentHandler class
class LIGOLWRAMContentHandler(ligolw.LIGOLWContentHandler):
startTable = ligolw.LIGOLWContentHandler.startTable
endTable = ligolw.LIGOLWContentHandler.endTable
from glue.ligolw import dbtables
class LIGOLWDBContentHandler(ligolw.LIGOLWContentHandler):
startTable = ligolw.LIGOLWContentHandler.startTable
endTable = ligolw.LIGOLWContentHandler.endTable
dbtables.DBTable.maxrowid = lambda self: None
#
# initialize the next_id attributes of all the table classes
#
#------------------------------------------------------------------
#from insert_nextid import next_id
# included below.
import MySQLdb
from MySQLdb.constants import ER
TABLE_NAME = "ligolwids"
create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME
select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME
insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME
update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME
lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME
unlock_sql = "UNLOCK TABLES"
def create_id_table(connection):
connection.cursor().execute(create_table_sql)
def db_get_next_id(connection, table_name, create_table=True, create_row=True):
cursor = connection.cursor()
try:
try:
cursor.execute(lock_sql)
cursor.execute(select_id_sql, [table_name])
next = cursor.fetchone()
if next is None:
cursor.execute(insert_new_sql, [table_name])
cursor.execute(select_id_sql, [table_name])
next = cursor.fetchone()
cursor.execute(update_sql, [table_name])
finally:
cursor.execute(unlock_sql)
cursor.close()
except MySQLdb.ProgrammingError, e:
if e.args[0] != ER.NO_SUCH_TABLE or not create_table:
raise
create_id_table(connection)
return db_get_next_id(connection, table_name, create_table=False)
return int(next[0])
#------------------------------------------------------------------
def initialize(connection):
for cls in lsctables.TableByName.values():
if cls.next_id is not None:
cls.get_next_id = lambda self: type(self.next_id)(db_get_next_id(connection, self.next_id.table_name))
classmethod(cls.get_next_id)
def insert_ligolw_tables(connection, filename, verbose=False):
initialize(connection)
#
# parse .xml file into memory, reassign ids, write xml stream to in-ram
# buffer
#
xmldoc = ligolw_reassign_ids(
utils.load_filename(
filename,
gz = (filename or "stdin").endswith(".gz"),
verbose = verbose,
contenthandler = LIGOLWRAMContentHandler
),
verbose = verbose
)
# <-- find coinc_event_id
coinc_table = glue.ligolw.table.get_table(
xmldoc,
glue.ligolw.lsctables.CoincTable.tableName)
coinc_id = coinc_table[0].coinc_event_id
buf = StringIO.StringIO()
utils.write_fileobj(xmldoc, buf)
buf.seek(0)
#
# re-parse xml stream from in-ram buffer to sqlite database
#
dbtables.DBTable_set_connection(connection)
xmldoc, digest = utils.load_fileobj(buf, contenthandler = LIGOLWDBContentHandler)
xmldoc.unlink()
dbtables.DBTable_set_connection(None)
return coinc_id
if __name__ == "__main__":
import sys
user = sys.argv[1]
password = sys.argv[2]
db = sys.argv[3]
filename = sys.argv[4]
conn= MySQLdb.connect(user=user, passwd=password, db=db)
rv = insert_ligolw_tables(conn, filename)
print "OK", rv
import MySQLdb
from MySQLdb.constants import ER
conn = MySQLdb.connect(db='ligolw', user='root')
TABLE_NAME = "ligolwids"
create_table_sql = "CREATE TABLE %s (tablename VARCHAR(30) PRIMARY KEY, nextid INTEGER)" % TABLE_NAME
select_id_sql = "SELECT nextid FROM %s WHERE tablename = %%s" % TABLE_NAME
insert_new_sql = "INSERT INTO %s (tablename, nextid) VALUES (%%s, 0)" % TABLE_NAME
update_sql = "UPDATE %s SET nextid = nextid+1 WHERE tablename = %%s" % TABLE_NAME
lock_sql = "LOCK TABLE %s WRITE" % TABLE_NAME
unlock_sql = "UNLOCK TABLES"
def create_id_table(connection):
connection.cursor().execute(create_table_sql)
def next_id(connection, table_name, create_table=True, create_row=True):
next = None
cursor = connection.cursor()
try:
try:
cursor.execute(lock_sql)
cursor.execute(select_id_sql, [table_name])
next = cursor.fetchone()
if not next:
cursor.execute(insert_new_sql, [table_name])
cursor.execute(select_id_sql, [table_name])
next = cursor.fetchone()
cursor.execute(update_sql, [table_name])
finally:
cursor.execute(unlock_sql)
cursor.close()
except MySQLdb.ProgrammingError, e:
# XXX remove
global foo
foo = e
cursor.close()
if e.args[0] == ER.NO_SUCH_TABLE:
if not create_table:
raise
create_id_table(connection)
return next_id(connection, table_name, create_table=False)
else:
raise
cursor.close()
return next[0]
# This is an auto-generated Django model module.
# You'll have to do the following manually to clean this up:
# * Rearrange models' order
# * Make sure each model has one field with primary_key=True
# Feel free to rename the models, but don't rename db_table values or field names.
#
# Also note:
# You'll have to insert the output of 'django-admin.py sqlcustom [appname]'
# into your database.
from django.db import models
class CoincDefiner(models.Model):
search = models.CharField(max_length=150, blank=True)
description = models.CharField(max_length=150, blank=True)
coinc_def_id = models.CharField(max_length=150, primary_key=True)
search_coinc_type = models.IntegerField(null=True, blank=True)
class Meta:
db_table = u'coinc_definer'
class TimeSlide(models.Model):
instrument = models.CharField(max_length=150, primary_key=True)
time_slide_id = models.CharField(max_length=150, primary_key=True)
process_id = models.CharField(max_length=150, blank=True)
offset = models.FloatField(null=True, blank=True)
class Meta:
db_table = u'time_slide'
class Process(models.Model):
program = models.CharField(max_length=150, blank=True)
version = models.CharField(max_length=150, blank=True)
cvs_repository = models.CharField(max_length=150, blank=True)
cvs_entry_time = models.IntegerField(null=True, blank=True)
comment = models.CharField(max_length=150, blank=True)
is_online = models.IntegerField(null=True, blank=True)
node = models.CharField(max_length=150, blank=True)
username = models.CharField(max_length=150, blank=True)
unix_procid = models.IntegerField(null=True, blank=True)
start_time = models.IntegerField(null=True, blank=True)
end_time = models.IntegerField(null=True, blank=True)
jobid = models.IntegerField(null=True, blank=True)
domain = models.CharField(max_length=150, blank=True)
ifos = models.CharField(max_length=150, blank=True)
process_id = models.CharField(max_length=150, primary_key=True)
class Meta:
db_table = u'process'
class CoincEvent(models.Model):
coinc_event_id = models.CharField(max_length=150, primary_key=True)
instruments = models.CharField(max_length=150, blank=True)
nevents = models.IntegerField(null=True, blank=True)
#process_id = models.CharField(max_length=150, blank=True)
process = models.ForeignKey(Process, blank=True)
#coinc_def_id = models.CharField(max_length=150, blank=True)
coinc_def = models.ForeignKey(CoincDefiner, blank=True)
#time_slide_id = models.CharField(max_length=150, blank=True)
time_slide = models.ForeignKey(TimeSlide, blank=True)
likelihood = models.FloatField(null=True, blank=True)
class Meta:
db_table = u'coinc_event'
class CoincEventMap(models.Model):
event_id = models.CharField(max_length=150, blank=True)
table_name = models.CharField(max_length=150, blank=True)
coinc_event_id = models.CharField(max_length=150, blank=True)
class Meta:
db_table = u'coinc_event_map'
class CoincInspiral(models.Model):
false_alarm_rate = models.FloatField(null=True, blank=True)
mchirp = models.FloatField(null=True, blank=True)
#coinc_event_id = models.CharField(max_length=150, blank=True)
coinc_event = models.OneToOneField(CoincEvent, primary_key=True)
combined_far = models.FloatField(null=True, blank=True)
mass = models.FloatField(null=True, blank=True)
end_time = models.IntegerField(null=True, blank=True)
snr = models.FloatField(null=True, blank=True)
end_time_ns = models.IntegerField(null=True, blank=True)
ifos = models.CharField(max_length=150, blank=True)
class Meta:
db_table = u'coinc_inspiral'
class Experiment(models.Model):
search = models.CharField(max_length=150, blank=True)
instruments = models.CharField(max_length=150, blank=True)
search_group = models.CharField(max_length=150, blank=True)
comments = models.CharField(max_length=150, blank=True)
lars_id = models.CharField(max_length=150, blank=True)
experiment_id = models.CharField(max_length=150, primary_key=True)
gps_start_time = models.IntegerField(null=True, blank=True)
gps_end_time = models.IntegerField(null=True, blank=True)
class Meta:
db_table = u'experiment'
class ExperimentMap(models.Model):
experiment_summ_id = models.CharField(max_length=150, blank=True)
#coinc_event_id = models.CharField(max_length=150, blank=True)
coinc_event = models.ForeignKey(CoincEvent, blank=True)
class Meta:
db_table = u'experiment_map'
class ExperimentSummary(models.Model):
sim_proc_id = models.CharField(max_length=150, blank=True)
datatype = models.CharField(max_length=150, blank=True)
experiment_summ_id = models.CharField(max_length=150, primary_key=True)
nevents = models.IntegerField(null=True, blank=True)
#experiment_id = models.CharField(max_length=150, blank=True)
experiment = models.ForeignKey(Experiment, blank=True)
duration = models.IntegerField(null=True, blank=True)
#time_slide_id = models.CharField(max_length=150, blank=True)
time_slide = models.ForeignKey(TimeSlide, blank=True)
veto_def_name = models.CharField(max_length=150, blank=True)
rowid = models.IntegerField(unique=True, db_column='ROWID') # Field name made lowercase.
class Meta:
db_table = u'experiment_summary'
class Ligolwids(models.Model):
tablename = models.CharField(max_length=90, primary_key=True)
nextid = models.IntegerField(null=True, blank=True)
class Meta:
db_table = u'ligolwids'
class ProcessParams(models.Model):
program = models.CharField(max_length=150, blank=True)
#process_id = models.CharField(max_length=150, blank=True)
process = models.ForeignKey(Process, blank=True)
param = models.CharField(max_length=150, blank=True)
type = models.CharField(max_length=150, blank=True)
value = models.CharField(max_length=150, blank=True)
class Meta:
db_table = u'process_params'
class SearchSummary(models.Model):
#process_id = models.CharField(max_length=150, blank=True)
process = models.ForeignKey(Process, blank=True)
shared_object = models.CharField(max_length=150, blank=True)
lalwrapper_cvs_tag = models.CharField(max_length=150, blank=True)
lal_cvs_tag = models.CharField(max_length=150, blank=True)
comment = models.CharField(max_length=150, blank=True)
ifos = models.CharField(max_length=150, blank=True)
in_start_time = models.IntegerField(null=True, blank=True)
in_start_time_ns = models.IntegerField(null=True, blank=True)
in_end_time = models.IntegerField(null=True, blank=True)
in_end_time_ns = models.IntegerField(null=True, blank=True)
out_start_time = models.IntegerField(null=True, blank=True)
out_start_time_ns = models.IntegerField(null=True, blank=True)
out_end_time = models.IntegerField(null=True, blank=True)
out_end_time_ns = models.IntegerField(null=True, blank=True)
nevents = models.IntegerField(null=True, blank=True)
nnodes = models.IntegerField(null=True, blank=True)
class Meta:
db_table = u'search_summary'
class SearchSummvars(models.Model):
#process_id = models.CharField(max_length=150, blank=True)
process = models.ForeignKey(Process, blank=True)
name = models.CharField(max_length=150, blank=True)
string = models.CharField(max_length=150, blank=True)
value = models.FloatField(null=True, blank=True)
search_summvar_id = models.CharField(max_length=150, primary_key=True)
class Meta:
db_table = u'search_summvars'
class SnglInspiral(models.Model):
process_id = models.CharField(max_length=150, blank=True)
ifo = models.CharField(max_length=150, blank=True)
search = models.CharField(max_length=150, blank=True)
channel = models.CharField(max_length=150, blank=True)
end_time = models.IntegerField(null=True, blank=True)
end_time_ns = models.IntegerField(null=True, blank=True)
end_time_gmst = models.FloatField(null=True, blank=True)
impulse_time = models.IntegerField(null=True, blank=True)
impulse_time_ns = models.IntegerField(null=True, blank=True)
template_duration = models.FloatField(null=True, blank=True)
event_duration = models.FloatField(null=True, blank=True)
amplitude = models.FloatField(null=True, blank=True)
eff_distance = models.FloatField(null=True, blank=True)
coa_phase = models.FloatField(null=True, blank=True)
mass1 = models.FloatField(null=True, blank=True)
mass2 = models.FloatField(null=True, blank=True)
mchirp = models.FloatField(null=True, blank=True)
mtotal = models.FloatField(null=True, blank=True)
eta = models.FloatField(null=True, blank=True)
kappa = models.FloatField(null=True, blank=True)
chi = models.FloatField(null=True, blank=True)
tau0 = models.FloatField(null=True, blank=True)
tau2 = models.FloatField(null=True, blank=True)
tau3 = models.FloatField(null=True, blank=True)
tau4 = models.FloatField(null=True, blank=True)
tau5 = models.FloatField(null=True, blank=True)
ttotal = models.FloatField(null=True, blank=True)
psi0 = models.FloatField(null=True, blank=True)
psi3 = models.FloatField(null=True, blank=True)
alpha = models.FloatField(null=True, blank=True)
alpha1 = models.FloatField(null=True, blank=True)
alpha2 = models.FloatField(null=True, blank=True)
alpha3 = models.FloatField(null=True, blank=True)
alpha4 = models.FloatField(null=True, blank=True)
alpha5 = models.FloatField(null=True, blank=True)
alpha6 = models.FloatField(null=True, blank=True)
beta = models.FloatField(null=True, blank=True)
f_final = models.FloatField(null=True, blank=True)
snr = models.FloatField(null=True, blank=True)
chisq = models.FloatField(null=True, blank=True)
chisq_dof = models.IntegerField(null=True, blank=True)
bank_chisq = models.FloatField(null=True, blank=True)
bank_chisq_dof = models.IntegerField(null=True, blank=True)
cont_chisq = models.FloatField(null=True, blank=True)
cont_chisq_dof = models.IntegerField(null=True, blank=True)
sigmasq = models.FloatField(null=True, blank=True)
rsqveto_duration = models.FloatField(null=True, blank=True)
gamma0 = models.FloatField(null=True, db_column='Gamma0', blank=True) # Field name made lowercase.
gamma1 = models.FloatField(null=True, db_column='Gamma1', blank=True) # Field name made lowercase.
gamma2 = models.FloatField(null=True, db_column='Gamma2', blank=True) # Field name made lowercase.
gamma3 = models.FloatField(null=True, db_column='Gamma3', blank=True) # Field name made lowercase.
gamma4 = models.FloatField(null=True, db_column='Gamma4', blank=True) # Field name made lowercase.
gamma5 = models.FloatField(null=True, db_column='Gamma5', blank=True) # Field name made lowercase.
gamma6 = models.FloatField(null=True, db_column='Gamma6', blank=True) # Field name made lowercase.
gamma7 = models.FloatField(null=True, db_column='Gamma7', blank=True) # Field name made lowercase.
gamma8 = models.FloatField(null=True, db_column='Gamma8', blank=True) # Field name made lowercase.
gamma9 = models.FloatField(null=True, db_column='Gamma9', blank=True) # Field name made lowercase.
event_id = models.CharField(max_length=150, primary_key=True)
class Meta:
db_table = u'sngl_inspiral'
......@@ -104,4 +104,5 @@ INSTALLED_APPS = (
'django.contrib.sessions',
'django.contrib.sites',
'gracedb.gracedb',
#'gracedb.ligolw',
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment