Skip to content
Snippets Groups Projects
Commit 736641fb authored by Tanner Prestegard's avatar Tanner Prestegard Committed by GraceDB
Browse files

adding new search app and search web page for events and superevents

parent d7abec11
No related branches found
No related tags found
1 merge request!8Superevents
......@@ -293,6 +293,7 @@ INSTALLED_APPS = [
'superevents',
'userprofile',
'ligoauth',
'search',
'rest_framework',
'guardian',
'django_twilio',
......@@ -503,6 +504,11 @@ LOGGING = {
'propagate': True,
'level': LOG_LEVEL,
},
'search': {
'handlers': ['debug_file','error_file'],
'propagate': True,
'level': LOG_LEVEL,
},
'userprofile': {
'handlers': ['debug_file','error_file'],
'propagate': True,
......
......@@ -13,6 +13,7 @@ from events.feeds import EventFeed, feedview
# After Django 1.10, have to import views directly, rather
# than just using a string
import events.views
import search.views
import events.reports
feeds = {
......@@ -38,6 +39,7 @@ urlpatterns = [
url(r'^latest', events.views.latest, name="latest"),
#(r'^reports/(?P<path>.+)$', 'django.views.static.serve',
# {'document_root': settings.LATENCY_REPORT_DEST_DIR}),
url(r'^search/$', search.views.search, name="mainsearch"),
# API URLs
url(r'^apiweb/', include('events.api.urls', app_name="api",
......@@ -50,7 +52,6 @@ urlpatterns = [
# Uncomment the admin/doc line below and add 'django.contrib.admindocs'
# to INSTALLED_APPS to enable admin documentation:
# (r'^admin/doc/', include('django.contrib.admindocs.urls')),
url(r'^admin/', admin.site.urls),
]
......
from django import forms
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe
from django.utils.html import escape
from events.models import Event
from events.query import parseQuery, filter_for_labels
from superevents.models import Superevent
from superevents.query import parseSupereventQuery
from pyparsing import ParseException
import os
import logging
logger = logging.getLogger(__name__)
htmlEntityStar = "&#9733;"
errorMarker = '<span style="color:red;">'+htmlEntityStar+'</span>'
class MainSearchForm(forms.Form):
QUERY_TYPE_CHOICES = (
('E', 'Event'),
('S', 'Superevent'),
)
FORMAT_CHOICES = (
('S', 'standard'),
('F', 'flexigrid'),
('L', 'ligolw'),
)
query = forms.CharField(required=False, widget=forms.TextInput(
attrs={'size': 60}))
query_type = forms.ChoiceField(required=True,
choices=QUERY_TYPE_CHOICES, label="Search for", initial='S')
get_neighbors = forms.BooleanField(required=False,
help_text="(Events only)")
results_format = forms.ChoiceField(required=False, initial='S',
choices=FORMAT_CHOICES, widget=forms.HiddenInput())
def clean(self):
# Do base class clean and just return if there are any errors already
cleaned_data = super(MainSearchForm, self).clean()
if self.errors:
return cleaned_data
# Get cleaned data
query_string = self.cleaned_data.get('query')
query_type = self.cleaned_data.get('query_type')
if query_type == 'S':
model = Superevent
parse_func = parseSupereventQuery
elif query_type == 'E':
model = Event
parse_func = parseQuery
# Don't need to check anything else here, can expect data to be good
# thanks to base class clean
# Parse query and get resulting objects
try:
qs = model.objects.filter(parse_func(query_string))
qs = filter_for_labels(qs,query_string).distinct()
cleaned_data['query'] = qs
return cleaned_data
except ParseException as e:
err = "Error: invalid query. (" + escape(e.pstr[:e.loc]) + \
errorMarker + escape(e.pstr[e.loc:]) + ")"
raise forms.ValidationError({'query': mark_safe(err)})
except Exception as e:
# What could this be and how can we handle it better? XXX
logger.error(e)
raise forms.ValidationError(str(e)+str(type(e)))
from django.http import HttpResponse, HttpResponseServerError, \
HttpResponseBadRequest
from events.view_utils import assembleLigoLw
from glue.ligolw import utils
RESULTS_LIMIT = 1000
def get_search_results_as_ligolw(objects):
if objects.count() > RESULTS_LIMIT:
return HttpResponseBadRequest(("Sorry -- no more than {0} events "
"currently allowed").format(RESULTS_LIMIT))
# Only valid for events, not superevents
if objects.model.__name__ == "Superevent":
return HttpResponseBadRequest("LigoLw tables are not available "
"for superevents")
try:
xmldoc = assembleLigoLw(objects)
except IOError as e:
msg = ("At least one of the query results has no associated coinc.xml "
"file. LigoLw tables are only available for queries which return "
"only coinc inspiral events. Please try your query again.")
return HttpResponseBadRequest(msg)
except Exception as e:
msg = ("An error occured while trying to compile LigoLw "
"results: {0}").format(e)
return HttpResponseServerError(msg)
response = HttpResponse(content_type='application/xml')
response['Content-Disposition'] = 'attachment; filename=gracedb-query.xml'
utils.write_fileobj(xmldoc, response)
return response
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
from django.utils.html import escape
from django.views.decorators.http import require_POST, require_GET
from .forms import MainSearchForm
from .utils import get_search_results_as_ligolw
from core.http import check_and_serve_file
from core.vfile import VersionedFile
from events.view_utils import flexigridResponse as events_flex
from superevents.search_flex import flexigridResponse as superevents_flex
import os
import logging
logger = logging.getLogger(__name__)
@require_GET
def search(request):
# Set up context
context = {}
if "query" in request.GET:
form = MainSearchForm(request.GET)
raw_query = request.GET['query']
if form.is_valid():
objects = form.cleaned_data.get('query')
query_type = form.cleaned_data.get('query_type')
get_neighbors = form.cleaned_data.get('get_neighbors')
_format = form.cleaned_data.get('results_format')
# TODO:
# Filter objects for user
# Get call from template for populating flexigrid table
if _format == 'F':
# Flex format
if query_type == 'S':
# Superevent query
flex_func = superevents_flex
elif query_type == 'E':
# Event query
flex_func = events_flex
else:
# TODO: raise error
pass
return flex_func(request, objects)
elif _format == 'L':
# LIGOLW format
return get_search_results_as_ligolw(objects)
context['title'] = "Query results"
context['objs'] = objects
context['raw_query'] = raw_query
context['query_type'] = query_type
context['get_neighbors'] = get_neighbors
else:
form = MainSearchForm()
# Update context
context['form'] = form
return render(request, 'search/query.html', context=context)
from django.conf import settings
from django.http import HttpResponse, HttpResponseBadRequest
from django.urls import reverse as django_reverse
from events.templatetags.scientific import scientific
from events.templatetags.timeutil import timeSelections
import os
import json
import logging
logger = logging.getLogger(__name__)
# XXX This should be configurable / moddable or something
MAX_QUERY_RESULTS = 1000
# The maximum number of rows to be returned by flexigridResponse
# in the event that the user asks for all of them.
MAX_FLEXI_ROWS = 250
def flexigridResponse(request, objects):
response = HttpResponse(content_type='application/json')
sortname = request.GET.get('sidx', None) # get index row - i.e. user click to sort
sortorder = request.GET.get('sord', 'desc') # get the direction
page = int(request.GET.get('page', 1)) # get the requested page
rp = int(request.GET.get('rows', 10)) # get how many rows we want to have into the grid
# select related objects to reduce the number of queries.
objects = objects.select_related('submitter', 'preferred_event')
objects = objects.prefetch_related('events', 'labels')
if sortname:
if sortorder == "desc":
sortname = "-" + sortname
objects = objects.order_by(sortname)
total = objects.count()
rows = []
if rp > -1:
start = (page-1) * rp
if total:
total_pages = (total / rp) + 1
else:
total_pages = 0
if page > total_pages:
page = total_pages
end = start+rp
else:
start = 0
total_pages = 1
page = 1
end = total-1
if total > MAX_FLEXI_ROWS:
return HttpResponseBadRequest("Too many rows! Please try loading a smaller number.")
for object in objects[start:end]:
t_start_times = timeSelections(object.t_start)
t_0_times = timeSelections(object.t_0)
t_end_times = timeSelections(object.t_end)
created_times = timeSelections(object.date_created)
cell_values = [
'<a href="{0}">{1}</a>'.format(
django_reverse("superevents:view", args=[
object.superevent_id]), object.superevent_id),
#Labels
" ".join(["""<span onmouseover="tooltip.show(tooltiptext('%s', '%s', '%s'));" onmouseout="tooltip.hide();" style="color: %s"> %s </span>""" % (label.label.name, label.creator.username, label.created, label.label.defaultColor, label.label.name) for label in object.labelling_set.all()]),
t_start_times.get('gps', ""),
t_0_times.get('gps', ""),
t_end_times.get('gps', ""),
str(object.is_gw),
'<a href="%s">Data</a>' % '#', # TODO: fix this #object.weburl(),
created_times.get('utc', ""),
"%s %s" % (object.submitter.first_name, object.submitter.last_name)
]
rows.append({
'id' : object.id,
'cell': cell_values,
})
d = {
'page': page,
'total': total_pages,
'records': total,
'rows': rows,
}
try:
msg = json.dumps(d)
except Exception:
# XXX Not right not right not right.
msg = "{}"
response['Content-length'] = len(msg)
response.write(msg)
return response
{% extends "base.html" %}
{% block bodyattrs %}
onload="document.search_form.query.focus();"
{% endblock %}
{% block title %}Search{% endblock %}
{% block heading %}Search for events or superevents{% endblock %}
{% block pageid %}search{% endblock %}
{% block jscript %}
{% load static %}
<link rel="stylesheet" type="text/css" media="screen" href="{% static "css/jqgrid/theme/jquery-ui.css" %}" />
<link rel="stylesheet" type="text/css" media="screen" href="{% static "css/jqgrid/theme/ui.all.css" %}" />
<link rel="stylesheet" type="text/css" media="screen" href="{% static "css/jqgrid/ui.jqgrid.css" %}" />
<link rel="stylesheet" href="{% static "css/labeltips.css" %}" />
<script src="{% static "js/jquery-1.3.2.min.js" %}" type="text/javascript"></script>
<script src="{% static "js/grid.locale-en.js" %}" type="text/javascript"></script>
<script src="{% static "js/jquery.jqGrid.min.js" %}" type="text/javascript"></script>
<script src="{% static "js/jquery-ui-1.7.2.custom.min.js" %}" type="text/javascript"></script>
<script type="text/javascript">
function toggle_visibility(id) {
var e = document.getElementById(id);
if(e.style.display == 'block')
e.style.display = 'none';
else
e.style.display = 'block';
}
</script>
<script type="text/javascript">
$(document).ready(function(){
$("#flex_superevents").jqGrid
(
{
sortable: true,
url: '{% url "mainsearch" %}?query={{raw_query|urlencode}}&query_type={{query_type}}&results_format=F',
datatype: 'json',
mtype: "GET",
colNames : ["UID", "Labels", "t_start", "t_0", "t_end", "is_gw", "Links", "Submitted", "Submitted By",
],
colModel : [
{name : 'id', index: 'id', width : 30, sortable : true, align: 'left', hidden: false},
{name : 'labels', index: 'labels', width : 40, sortable : false, align: 'left'},
{name : 't_start', t_start: 't_start', width : 20, sortable : false, align: 'left', hidden: false},
{name : 't_0', index: 't_0', width : 20, sortable : false, align: 'left', hidden: false},
{name : 't_end', index: 't_end', width : 20, sortable : false, align: 'left', hidden: false},
{name : 'is_gw', index: 'is_gw', width : 20, sortable : true, align: 'left'},
{name : 'links', index: 'links', width : 30, sortable : false, align: 'center'},
{name : 'created', index: 'created', width : 40, sortable : true, align: 'left'},
{name : 'submitter', index: 'submitter', width : 60, sortable : true, align: 'left'},
],
pager: '#pager1',
rowNum:20,
rowList:[10,20,30,40,50,75,100,-1],
sortname: 'id',
sortorder: "desc",
viewrecords: true,
caption: 'Query Results',
//forceFit: true,
//height: '150px',
height: 'auto',
//loadui: 'enable', // block, disable
toolbar: [true, "top"],
autowidth: true,
loadError: function (jqXHR, textStatus, errorThrown) {
alert('Error, ' + jqXHR.status + ': ' + jqXHR.responseText);
},
}
);
//$("#flex_superevents").jqGrid('gridResize',{});
//$("#flex_superevents").jqGrid('gridResize',{ minWidth:350,maxWidth:800,minHeight:80, maxHeight:350 });
$("#flex_superevents").jqGrid('navGrid','#pager1',{search: false, edit:false,add:false,del:false});
$("#t_flex_superevents").append("<input type='button' value='Select Columns' style='height:20px;font-size:-3'/>");
$("input","#t_flex_superevents").click(function(){
$("#flex_superevents").jqGrid('setColumns');
});
});
$(document).ready(function(){
$("#flex_events").jqGrid
(
{
sortable: true,
url: '{% url "mainsearch" %}?query={{raw_query|urlencode}}&query_type={{query_type}}&get_neighbors={{get_neighbors}}&results_format=F',
datatype: 'json',
mtype: "GET",
colNames : ["UID", "Labels", {% if get_neighbors %} "Neighbors (+/-5sec)", {% endif %} "Group", "Pipeline", "Search", "Event Time", "Instruments", "FAR (Hz)", "Links", "Submitted", "Submitted By",
],
colModel : [
{name : 'id', index: 'id', width : 50, sortable : true, align: 'left', hidden: false},
{name : 'labels', index: 'labels', width : 70, sortable : false, align: 'left'},
{% if get_neighbors %}
{name : 'neighbors', index: 'neighbors', width : 100, sortable : false, align: 'left'},
{% endif %}
{name : 'group', index: 'group', width : 50, sortable : false, align: 'left', hidden: true},
{name : 'pipeline', index: 'pipeline', width : 50, sortable : false, align: 'left', hidden: true},
{name : 'search', index: 'search', width : 50, sortable : false, align: 'left', hidden: true},
{name : 'gpstime', index: 'gpstime', width : 70, sortable : true, align: 'right'},
{name : 'instruments', index: 'instruments', width : 50, sortable : true, align: 'right', hidden: false},
{name : 'far', index: 'far', width : 80, sortable : true, align: 'right', hidden: false},
{name : 'links', index: 'links', width : 80, sortable : false, align: 'center'},
{name : 'created', index: 'created', width : 100, sortable : true, align: 'right'},
{name : 'submitter', index: 'submitter', width : 80, sortable : true, align: 'right'},
],
pager: '#pager1',
rowNum:20,
rowList:[10,20,30,40,50,75,100,-1],
sortname: 'id',
sortorder: "desc",
viewrecords: true,
caption: 'Query Results',
//forceFit: true,
//height: '150px',
height: 'auto',
//loadui: 'enable', // block, disable
toolbar: [true, "top"],
autowidth: true,
loadError: function (jqXHR, textStatus, errorThrown) {
alert('Error, ' + jqXHR.status + ': ' + jqXHR.responseText);
},
}
);
//$("#flex_events").jqGrid('gridResize',{});
//$("#flex_events").jqGrid('gridResize',{ minWidth:350,maxWidth:800,minHeight:80, maxHeight:350 });
$("#flex_events").jqGrid('navGrid','#pager1',{search: false, edit:false,add:false,del:false});
$("#t_flex_events").append("<input type='button' value='Select Columns' style='height:20px;font-size:-3'/>");
$("input","#t_flex_events").click(function(){
$("#flex_events").jqGrid('setColumns');
});
});
</script>
{% endblock %}
{% block content %}
<form method="GET" name="search_form">
<table>
{{ form.as_table }}
<tr><td></td><td><input type="Submit" value="Search" class="searchButtonClass"></td></tr>
<tr><td></td><td><a onClick="toggle_visibility('hints');">Query help</a>
| <a href="{% url "search" %}">Link to old search page</a>
{% if raw_query %}
| <a href="{{ request.build_absolute_uri }}">Link to current query</a>
{% if query_type == 'E' %}
| <a href="{% url "mainsearch" %}?query={{raw_query|urlencode}}&query_type={{query_type}}&get_neighbors={{get_neighbors}}&results_format=L">Download LIGOLW file</a>
{% endif %}
{% endif %}
</td></tr>
<tr><td></td><td>{% include "search/query_help_frag.html" %}</td></tr>
</table>
</form>
<!-- Search results table -->
{% if query_type == 'S' %}
<table id="flex_superevents"></table>
{% elif query_type == 'E' %}
<table id="flex_events"></table>
{% endif %}
<div id="pager1"></div>
{% endblock %}
<div id="hints" style="display: none;">
<p><b>NOTE:</b> Clicking the 'Get neighbors' checkbox will result in an additional neighbors query for each item in the search results, and the neighbors are thus shown in the results table. However, this causes the overall query to take longer, which is why it is un-checked by default. </p>
<h2>Event queries</h2>
<h4>By event attributes</h4>
Relational and range queries can be made on event attributes.
<ul>
<li><code>instruments = "H1,L1,V1" &amp; far &lt; 1e-7</code></li>
<li><code>singleinspiral.mchirp &gt;= 0.5 &amp; singleinspiral.eff_distance in 0.0,55 </code></li>
<li><code>(si.channel = "DMT-STRAIN" | si.channel = "DMT-PAIN") &amp; si.snr &lt; 5</code></li>
<li><code>mb.snr in 1,3 &amp; mb.central_freq &gt; 1000</code></li>
</ul>
Attributes in the common event object (eg gpstime, far, instruments) do not need qualifiers. Attributes specific to inspiral or burst events, for example, require qualification. Abbreviations are available: <code>si</code> for singleinspiral, <code>ci</code> for coincinspiral and <code>mb</code> for multiburst.
<h4>By GPS time</h4>
Specify an exact GPS time, or a range.
Integers will be assumed to be GPS times, making the <code>gpstime:</code>
keyword optional.
<ul>
<li><code>899999000 .. 999999999</code></li>
<li><code>gpstime: 899999000.0 .. 999999999.9</code></li>
</ul>
<h4>By creation time</h4>
Creation time may be indicated by an exact time or a range. Date/times are
in the format <code>2009-10-20 13:00:00</code> (must be UTC). If the time is omitted, it
is assumed to be <code>00:00:00</code>. Dates may also consist of certain
variants of English-like phrases.
The <code>created:</code> keyword is (generally) optional.
<ul>
<li><code>created: 2009-10-08 .. 2009-12-04 16:00:00</code></li>
<li><code>yesterday..now</code></li>
<li><code>created: 1 week ago .. now</code></li>
</ul>
<h4>By graceid</h4>
GraceIds can be specified either individually, or as a range.
The <code>gid:</code> keyword is optional.
<ul>
<li>gid: G2011</li>
<li>G2011 .. G3000</li>
<li>G2011 G2032 G2033</li>
</ul>
<h4>By group, pipeline, and search</h4>
The <code>group:</code>, <code>pipeline:</code>, and <code>search:</code> keywords are optional. Names are case-insensitive. Note that events in the Test group will not be shown unless explicitly requested.
<ul>
<li>CBC Burst</li>
<!-- <li>Inspiral CWB</li> -->
<li>group: Test pipeline: cwb</li>
<li>Burst cwb</li>
</ul>
<h4>By label</h4>
You may request only events with a particular label or set of labels. The <code>label:</code> keyword
is optional. Label names can be combined with binary AND: '&amp;' or ','; or binary OR: '|'. For N labels,
there must be exactly N-1 binary operators. (Parentheses are not allowed.) Additionally, any of the labels
in a query string can be negated with '~' or '-'.
<ul>
<li>label: INJ</li>
<li>EM_READY &amp; ADVOK</li>
<li>H1OK | L1OK &amp; ~INJ &amp; ~DQV</li>
</ul>
Labels in current use are:
INJ, DQV, EM_READY, PE_READY, H1NO, H1OK, H1OPS, L1NO, L1OK, L1OPS, V1NO, V1OK, V1OPS, ADVREQ, ADVOK, ADVNO, EM_COINC, EM_SENT
<h4>By submitter</h4>
To specify events from a given submitter, indicate the name of the submitter in double quotes.
The <code>submitter:</code> is optional. While LIGO user names are predictable, most events
are submitted through robot accounts and are not as predictable. This is probably a defect
that ought to be remedied.
<ul>
<li>"waveburst"
<li>"gstlalcbc" "gracedb.processor"
<li>submitter: "joss.whedon@ligo.org" submitter: "gracedb.processor"
</ul>
<h4>By superevent status</h4>
Use the <code>in_superevent:</code> to specify events which are/are not part of any superevent.
Use the <code>superevent:</code> keyword to specify events which are part of a specific superevent.
Use the <code>is_preferred_event:</code> keyword to specify events which are/are not preferred events for any superevent.
<ul>
<li>in_superevent: True</li>
<li>in_superevent: False</li>
<li>superevent: S180525c</li>
<li>is_preferred_event: True</li>
<li>is_preferred_event: False</li>
</ul>
<br />
<h2>Superevent queries</h2>
<h4>By id</h4>
The keywords <code>id:</code> or <code>superevent_id:</code> are optional.
<ul>
<li>id: S180525</li>
<li>superevent_id: S170817b</li>
<li>GW180428C</li>
</ul>
<h4>By GPS time</h4>
Same as for event queries, with keywords <code>gpstime:</code> or <code>t_0:</code>.
<h4>By other time attributes</h4>
Queries based on the <code>t_start</code> and <code>t_end</code> attributes are also available.
<ul>
<li>t_start: 899999000</li>
<li>t_end: 899999000.0 .. 900000000.0</li>
</ul>
<h4>By preferred event graceids</h4>
Specify a graceid or range of graceids with keyword <code>preferred_event:</code> to get all
superevents with corresponding preferred events.
<ul>
<li>preferred_event: G123456</li>
<li>preferred_event: G123456 .. G123500</li>
</ul>
<h4>By event graceids</h4>
Specify a graceid or range of graceids with keyword <code>event:</code> to get all
superevents which contain the corresponding event(s).
<ul>
<li>event: G123456</li>
<li>event: G123456 .. G123500</li>
</ul>
<h4>By GW status</h4>
Query for superevents which are confirmed as GWs or not with the <code>is_gw:</code> keyword.
<ul>
<li>is_gw: True</li>
<li>is_gw: False</li>
</ul>
<h4>By creation time</h4>
Same as for events.
<h4>By submitter</h4>
Same as for events.
</div>
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment