From 600194e98505cd1621284818a41e9b65e9416ac7 Mon Sep 17 00:00:00 2001
From: Tanner Prestegard <tanner.prestegard@ligo.org>
Date: Fri, 25 May 2018 10:30:59 -0500
Subject: [PATCH] reorganizing queries and query utilities for events
---
gracedb/events/fields.py | 46 ++++++++++++++++++++++
gracedb/events/forms.py | 22 +----------
gracedb/events/query.py | 63 +++---------------------------
gracedb/events/query_utils.py | 72 +++++++++++++++++++++++++++++++++++
4 files changed, 125 insertions(+), 78 deletions(-)
create mode 100644 gracedb/events/fields.py
create mode 100644 gracedb/events/query_utils.py
diff --git a/gracedb/events/fields.py b/gracedb/events/fields.py
new file mode 100644
index 000000000..8ba979934
--- /dev/null
+++ b/gracedb/events/fields.py
@@ -0,0 +1,46 @@
+
+from django import forms
+from django.utils.safestring import mark_safe
+from django.utils.html import escape
+from django.contrib.auth.models import User
+from django.core.exceptions import FieldError
+from django.forms import ModelForm
+from django.db.models import Q
+
+from .models import Event, Group, Label
+from .models import Pipeline, Search, Signoff
+from .query import parseQuery, filter_for_labels
+
+from pyparsing import ParseException
+
+htmlEntityStar = "★"
+htmlEntityRightPointingHand = "☞"
+htmlEntitySkullAndCrossbones = "☠"
+htmlEntityTriangularBuller = "‣"
+htmlEntityRightArrow = "→"
+errorMarker = '<span style="color:red;">'+htmlEntityStar+'</span>'
+
+class GraceQueryField(forms.CharField):
+
+ def do_filtering(self, query_string):
+ """Method for getting queryset based on a query string"""
+ qs = Event.objects.filter(parseQuery(query_string))
+ qs = filter_for_labels(qs, query_string)
+ return qs
+
+ def clean(self, queryString):
+ queryString = forms.CharField.clean(self, queryString)
+ try:
+ qs = self.do_filtering(queryString)
+ return qs.distinct()
+ except ParseException, e:
+ err = "Error: " + escape(e.pstr[:e.loc]) + errorMarker + escape(e.pstr[e.loc:])
+ raise forms.ValidationError(mark_safe(err))
+ except FieldError, e:
+ # XXX error message can be more polished than this
+ err = "Error: " + str(e)
+ raise forms.ValidationError(mark_safe(err))
+ except Exception, e:
+ # What could this be and how can we handle it better? XXX
+ raise forms.ValidationError(str(e)+str(type(e)))
+
diff --git a/gracedb/events/forms.py b/gracedb/events/forms.py
index c986b5a50..5fdb51036 100644
--- a/gracedb/events/forms.py
+++ b/gracedb/events/forms.py
@@ -1,4 +1,3 @@
-
from django import forms
from django.utils.safestring import mark_safe
from django.utils.html import escape
@@ -8,6 +7,7 @@ from django.contrib.auth.models import User
from django.core.exceptions import FieldError
from django.forms import ModelForm
+from .fields import GraceQueryField
from .query import parseQuery, filter_for_labels
from pyparsing import ParseException
@@ -19,26 +19,6 @@ htmlEntityRightArrow = "→"
errorMarker = '<span style="color:red;">'+htmlEntityStar+'</span>'
-class GraceQueryField(forms.CharField):
- def clean(self, queryString):
- from django.db.models import Q
- queryString = forms.CharField.clean(self, queryString)
- try:
- #return Event.objects.filter(parseQuery(queryString)).distinct()
- qs = Event.objects.filter(parseQuery(queryString))
- qs = filter_for_labels(qs, queryString)
- return qs.distinct()
- except ParseException, e:
- err = "Error: " + escape(e.pstr[:e.loc]) + errorMarker + escape(e.pstr[e.loc:])
- raise forms.ValidationError(mark_safe(err))
- except FieldError, e:
- # XXX error message can be more polished than this
- err = "Error: " + str(e)
- raise forms.ValidationError(mark_safe(err))
- except Exception, e:
- # What could this be and how can we handle it better? XXX
- raise forms.ValidationError(str(e)+str(type(e)))
-
class SimpleSearchForm(forms.Form):
query = GraceQueryField(required=False, widget=forms.TextInput(attrs={'size':60}))
get_neighbors = forms.BooleanField(required=False)
diff --git a/gracedb/events/query.py b/gracedb/events/query.py
index 355ab4ae1..384770dcf 100644
--- a/gracedb/events/query.py
+++ b/gracedb/events/query.py
@@ -14,10 +14,11 @@
# (weak) natural language time parsing.
from .nltime import nlTimeExpression as nltime_
nltime = nltime_.setParseAction(lambda toks: toks["calculatedTime"])
+from .models import Group, Pipeline, Search, Label
+from .query_utils import maybeRange, getLabelQ, RUN_MAP
#import time, datetime
import datetime
-from .models import Group, Pipeline, Search, Label
from django.db.models import Q
from django.db.models.query import QuerySet
import pytz
@@ -28,14 +29,6 @@ from pyparsing import Word, nums, Literal, CaselessLiteral, delimitedList, \
oneOf, stringStart, stringEnd, FollowedBy, ParseResults, ParseException, \
CaselessKeyword
-def maybeRange(name, dbname=None):
- dbname = dbname or name
- def f(toks):
- if len(toks) == 1:
- return name, Q(**{dbname: toks[0]})
- return name, Q(**{dbname+"__range": toks.asList()})
- return f
-
def convertToGps(dateStr):
return 12
@@ -56,40 +49,11 @@ gpsQ = Optional(Suppress(Keyword("gpstime:"))) + (gpstime^gpstimeRange)
gpsQ = gpsQ.setParseAction(maybeRange("gpstime"))
# run ids
-runmap = {
- # 30 Nov 2016 16:00:00 UTC - 25 Aug 2017 22:00:00 UTC
- "O2" : (1164556817, 1187733618),
- # Friday, Sept 18th, 10 AM CDT 2015 - Tuesday, Jan 12th, 10:00 AM CST 2016
- "O1" : (1126623617, 1136649617),
- # Monday, Aug 17th, 10 AM CDT - Friday, Sept 18th, 10 AM CDT
- "ER8" : (1123858817, 1126623617),
- # Jun 03 21:00:00 UTC 2015 - Jun 14 15:00:00 UTC 2015
- "ER7" : (1117400416, 1118329216),
- # Dec 08 16:00:00 UTC 2014 - Dec 17 15:00:00 UTC 2014
- "ER6" : (1102089616, 1102863616),
- # Jan 15 12:00:00 UTC 2014 - Mar 15 2014 00:00:00 UTC
- "ER5" : (1073822416, 1078876816),
- # Jul 15 00:00:00 UTC 2013 - Aug 30 2013 00:00:00 UTC
- "ER4" : (1057881616, 1061856016),
- # Feb 5 16:00:00 CST 2013 - Mon Feb 25 00:00:00 GMT 2013
- "ER3" : (1044136816, 1045785616),
- # Jul 18 17:00:00 GMT 2012 - Aug 8 17:00:00 GMT 2012
- "ER2" : (1026666016, 1028480416),
- #"ER2" : (1026061216, 1028480416),
- #"ER2" : (1026069984, 1028480416), # soft start
- "ER1": (1011601640, 1013299215),
- "ER1test": (1010944815, 1011601640), # Pre ER1
- "S6" : (931035296, 971622087),
- "S6A" : (931035296, 935798487),
- "S6B" : (937800015, 947260815),
- "S6C" : (949449543, 961545687),
- "S6D" : (956707143, 971622087),
-}
-runid = Or(map(CaselessLiteral, runmap.keys())).setName("run id")
+runid = Or(map(CaselessLiteral, RUN_MAP.keys())).setName("run id")
#runidList = OneOrMore(runid).setName("run id list")
runQ = (Optional(Suppress(Keyword("runid:"))) + runid)
runQ = runQ.setParseAction(lambda toks: ("gpstime", Q(gpstime__range=
- runmap[toks[0]])))
+ RUN_MAP[toks[0]])))
# Gracedb ID
gid = Suppress(Word("gG", exact=1)) + Word("0123456789")
@@ -319,23 +283,8 @@ def parseQuery(s):
searchQ = searchQ.setParseAction(lambda toks:
("search", Q(search__name__in=toks.asList())))
- # labelQ is defined inside in order to avoid a compile-time database query
- # to get the label names.
- # Note the parse action for labelQ: Replace all tokens with the empty
- # string. This basically has the effect of removing any label query terms
- # from the query string.
- labelNames = [l.name for l in Label.objects.all()]
- #label = Or([CaselessLiteral(n) for n in labelNames]).\
- label = Or([CaselessKeyword(n) for n in labelNames]).\
- setParseAction( lambda toks: Q(labels__name=toks[0]) )
- andop = oneOf(", &")
- orop = Literal("|")
- minusop = oneOf("- ~")
- op = Or([andop,orop,minusop])
- oplabel = OneOrMore(op) + label
- labelQ_ = Optional(minusop) + label + ZeroOrMore(oplabel)
- labelQ = (Optional(Suppress(Keyword("label:"))) + labelQ_.copy())
- labelQ.setParseAction(lambda toks: '')
+ # Get labelQ
+ labelQ = getLabelQ()
# Clean the label-related parts of the query out of the query string.
s = labelQ.transformString(s)
diff --git a/gracedb/events/query_utils.py b/gracedb/events/query_utils.py
new file mode 100644
index 000000000..4d475dc4e
--- /dev/null
+++ b/gracedb/events/query_utils.py
@@ -0,0 +1,72 @@
+# (weak) natural language time parsing.
+from django.db.models import Q
+
+from .models import Label
+
+from pyparsing import Keyword, CaselessKeyword, oneOf, Literal, Or, \
+ OneOrMore, ZeroOrMore, Optional, Suppress
+
+
+# Dict of LIGO run names (keys) and GPS time range tuples (values)
+RUN_MAP = {
+ # 30 Nov 2016 16:00:00 UTC - 25 Aug 2017 22:00:00 UTC
+ "O2" : (1164556817, 1187733618),
+ # Friday, Sept 18th, 10 AM CDT 2015 - Tuesday, Jan 12th, 10:00 AM CST 2016
+ "O1" : (1126623617, 1136649617),
+ # Monday, Aug 17th, 10 AM CDT - Friday, Sept 18th, 10 AM CDT
+ "ER8" : (1123858817, 1126623617),
+ # Jun 03 21:00:00 UTC 2015 - Jun 14 15:00:00 UTC 2015
+ "ER7" : (1117400416, 1118329216),
+ # Dec 08 16:00:00 UTC 2014 - Dec 17 15:00:00 UTC 2014
+ "ER6" : (1102089616, 1102863616),
+ # Jan 15 12:00:00 UTC 2014 - Mar 15 2014 00:00:00 UTC
+ "ER5" : (1073822416, 1078876816),
+ # Jul 15 00:00:00 UTC 2013 - Aug 30 2013 00:00:00 UTC
+ "ER4" : (1057881616, 1061856016),
+ # Feb 5 16:00:00 CST 2013 - Mon Feb 25 00:00:00 GMT 2013
+ "ER3" : (1044136816, 1045785616),
+ # Jul 18 17:00:00 GMT 2012 - Aug 8 17:00:00 GMT 2012
+ "ER2" : (1026666016, 1028480416),
+ #"ER2" : (1026061216, 1028480416),
+ #"ER2" : (1026069984, 1028480416), # soft start
+ "ER1": (1011601640, 1013299215),
+ "ER1test": (1010944815, 1011601640), # Pre ER1
+ "S6" : (931035296, 971622087),
+ "S6A" : (931035296, 935798487),
+ "S6B" : (937800015, 947260815),
+ "S6C" : (949449543, 961545687),
+ "S6D" : (956707143, 971622087),
+}
+
+
+def maybeRange(name, dbname=None):
+ dbname = dbname or name
+ def f(toks):
+ if len(toks) == 1:
+ return name, Q(**{dbname: toks[0]})
+ return name, Q(**{dbname+"__range": toks.asList()})
+ return f
+
+
+def getLabelQ():
+ # labelQ is defined inside in order to avoid a compile-time database query
+ # to get the label names.
+ # Note the parse action for labelQ: Replace all tokens with the empty
+ # string. This basically has the effect of removing any label query terms
+ # from the query string.
+ labelNames = [l.name for l in Label.objects.all()]
+ #label = Or([CaselessLiteral(n) for n in labelNames]).\
+ label = Or([CaselessKeyword(n) for n in labelNames]).\
+ setParseAction( lambda toks: Q(labels__name=toks[0]) )
+ andop = oneOf(", &")
+ orop = Literal("|")
+ minusop = oneOf("- ~")
+ op = Or([andop,orop,minusop])
+ oplabel = OneOrMore(op) + label
+ labelQ_ = Optional(minusop) + label + ZeroOrMore(oplabel)
+ labelQ = (Optional(Suppress(Keyword("label:"))) + labelQ_.copy())
+ labelQ.setParseAction(lambda toks: '')
+
+ return labelQ
+
+
--
GitLab