From 94a0d50a51ccdf15df44496e9fb2f4b9e9ad4d7b Mon Sep 17 00:00:00 2001
From: Branson Stephens <branson.stephens@ligo.org>
Date: Fri, 29 Jan 2016 13:48:18 -0600
Subject: [PATCH] Changes required for USE_TZ = True. Times in db now are in
UTC. Localize EMFootprint start times to UTC. Changed event creation time
query to assume UTC.
---
gracedb/buildVOEvent.py | 4 +--
.../commands/create_test_perms_fixtures.py | 5 ++--
.../management/commands/make_histograms.py | 8 +++---
gracedb/models.py | 6 +++--
gracedb/nltime.py | 26 ++++++++++++-------
gracedb/query.py | 3 ++-
gracedb/reports.py | 14 +++++-----
gracedb/templatetags/flash.py | 5 ++--
gracedb/templatetags/timeutil.py | 17 +++++++-----
gracedb/view_logic.py | 14 ++++++++--
gracedb/view_utils.py | 9 ++++---
ligoauth/middleware/auth.py | 4 +--
settings/default.py | 8 ++++++
templates/gracedb/query_help_frag.html | 2 +-
userprofile/views.py | 4 +--
15 files changed, 83 insertions(+), 46 deletions(-)
diff --git a/gracedb/buildVOEvent.py b/gracedb/buildVOEvent.py
index c62b04f6d..9e3e3bdbc 100755
--- a/gracedb/buildVOEvent.py
+++ b/gracedb/buildVOEvent.py
@@ -21,7 +21,7 @@ from VOEventLib.VOEvent import Time, TimeInstant
# XXX ER2.utils. utils is in project directory. ugh.
from utils import gpsToUtc
-from datetime import datetime
+from django.utils import timezone
from django.conf import settings
from django.core.urlresolvers import reverse
from models import CoincInspiralEvent, MultiBurstEvent
@@ -104,7 +104,7 @@ def buildVOEvent(event, serial_number, voevent_type, request=None, skymap_filena
a.add_contactName("LIGO Scientific Collaboration and Virgo Collaboration")
#a.add_contactEmail("postmaster@ligo.org")
w.set_Author(a)
- w.set_Date(datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"))
+ w.set_Date(timezone.now().strftime("%Y-%m-%dT%H:%M:%S"))
v.set_Who(w)
############ Why ############################
diff --git a/gracedb/management/commands/create_test_perms_fixtures.py b/gracedb/management/commands/create_test_perms_fixtures.py
index 08f6ef283..398df14f1 100644
--- a/gracedb/management/commands/create_test_perms_fixtures.py
+++ b/gracedb/management/commands/create_test_perms_fixtures.py
@@ -1,7 +1,6 @@
import os
import json
from StringIO import StringIO
-from datetime import datetime
from gracedb.models import GrbEvent, Tag, Event
from gracedb.models import MultiBurstEvent
@@ -13,6 +12,8 @@ from django.core.management import call_command
from django.core.management.base import NoArgsCommand
from django.conf import settings
+from django.utils import timezone
+
#------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------
# Parameters
@@ -143,7 +144,7 @@ def get_user_field_dicts(user_info):
user_dict['user_permissions'] = []
user_dict['password'] = 'X'
- now = datetime.now().isoformat().split('.')[0]
+ now = timezone.now().isoformat().split('.')[0]
user_dict['last_login'] = now
user_dict['date_joined'] = now
diff --git a/gracedb/management/commands/make_histograms.py b/gracedb/management/commands/make_histograms.py
index 057e6348e..d36307de4 100644
--- a/gracedb/management/commands/make_histograms.py
+++ b/gracedb/management/commands/make_histograms.py
@@ -10,8 +10,8 @@ import numpy
from gracedb.models import Event, Pipeline
import os
-from datetime import datetime, timedelta
-
+from datetime import timedelta
+from django.utils import timezone
DEST_DIR = settings.LATENCY_REPORT_DEST_DIR
MAX_X = settings.LATENCY_MAXIMUM_CHARTED
@@ -27,7 +27,7 @@ class Command(NoArgsCommand):
def handle_noargs(self, **options):
- now = datetime.now()
+ now = timezone.now()
start_day = now - timedelta(1)
start_week = now - timedelta(7)
@@ -71,7 +71,7 @@ class Command(NoArgsCommand):
def writeIndex(notes, fname):
- createdDate = str(datetime.now())
+ createdDate = str(timezone.now())
maxx = MAX_X
table = '<table border="1" bgcolor="white">'
diff --git a/gracedb/models.py b/gracedb/models.py
index c2343ddf8..d328d5761 100644
--- a/gracedb/models.py
+++ b/gracedb/models.py
@@ -26,7 +26,8 @@ import json
from utils import posixToGpsTime
from django.conf import settings
-import pytz, time
+import pytz
+import calendar
from cStringIO import StringIO
from hashlib import sha1
@@ -175,7 +176,8 @@ class Event(models.Model):
dt = self.created
if not dt.tzinfo:
dt = SERVER_TZ.localize(dt)
- posix_time = time.mktime(dt.timetuple())
+ dt = dt.astimezone(pytz.utc)
+ posix_time = calendar.timegm(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
return gps_time - self.gpstime
diff --git a/gracedb/nltime.py b/gracedb/nltime.py
index b6ce811d0..0cbcc7338 100755
--- a/gracedb/nltime.py
+++ b/gracedb/nltime.py
@@ -6,6 +6,12 @@
from datetime import datetime, timedelta
from pyparsing import *
import calendar
+from django.utils import timezone
+import pytz
+
+# Note, since the 'now' comes from django.utils.timezone, it will be in UTC.
+# We should therefore localize all of the datetime objects generated here to
+# UTC.
# string conversion parse actions
def convertToTimedelta(toks):
@@ -25,7 +31,7 @@ def convertToTimedelta(toks):
toks["timeOffset"] = td
def convertToDay(toks):
- now = datetime.now()
+ now = timezone.now()
if "wkdayRef" in toks:
todaynum = now.weekday()
daynames = [n.lower() for n in calendar.day_name]
@@ -34,23 +40,23 @@ def convertToDay(toks):
daydiff = (nameddaynum + 7 - todaynum) % 7
else:
daydiff = -((todaynum + 7 - nameddaynum) % 7)
- toks["absTime"] = datetime(now.year, now.month, now.day)+timedelta(daydiff)
+ toks["absTime"] = pytz.utc.localize(datetime(now.year, now.month, now.day)+timedelta(daydiff))
else:
name = toks.name.lower()
toks["absTime"] = {
"now" : now,
- "today" : datetime(now.year, now.month, now.day),
- "yesterday" : datetime(now.year, now.month, now.day)+timedelta(-1),
- "tomorrow" : datetime(now.year, now.month, now.day)+timedelta(+1),
+ "today" : pytz.utc.localize(datetime(now.year, now.month, now.day)),
+ "yesterday" : pytz.utc.localize(datetime(now.year, now.month, now.day)+timedelta(-1)),
+ "tomorrow" : pytz.utc.localize(datetime(now.year, now.month, now.day)+timedelta(+1)),
}[name]
def convertToAbsTime(toks):
- now = datetime.now()
+ now = timezone.now()
if "dayRef" in toks:
day = toks.dayRef.absTime
- day = datetime(day.year, day.month, day.day)
+ day = pytz.utc.localize(datetime(day.year, day.month, day.day))
else:
- day = datetime(now.year, now.month, now.day)
+ day = pytz.utc.localize(datetime(now.year, now.month, now.day))
if "timeOfDay" in toks:
if isinstance(toks.timeOfDay,basestring):
timeOfDay = {
@@ -78,7 +84,7 @@ def calculateTime(toks):
if toks.absTime:
absTime = toks.absTime
else:
- absTime = datetime.now()
+ absTime = timezone.now()
if toks.timeOffset:
absTime += toks.timeOffset
toks["calculatedTime"] = absTime
@@ -181,7 +187,7 @@ if __name__ == "__main__":
2009/12/22 12:13:14""".splitlines()
for t in tests:
- print t, "(relative to %s)" % datetime.now()
+ print t, "(relative to %s)" % timezone.now()
res = nlTimeExpression.parseString(t)
if "calculatedTime" in res:
print res.calculatedTime
diff --git a/gracedb/query.py b/gracedb/query.py
index 5749f7515..f5c00fe9e 100644
--- a/gracedb/query.py
+++ b/gracedb/query.py
@@ -20,6 +20,7 @@ import datetime
import models
from django.db.models import Q
from django.db.models.query import QuerySet
+import pytz
from pyparsing import \
Word, nums, Literal, CaselessLiteral, delimitedList, Suppress, QuotedString, \
@@ -170,7 +171,7 @@ nltimeRange = nltime + Suppress("..") + nltime
def doTime(tok):
x = datetime.datetime(*(map(int, tok)))
- return x
+ return pytz.utc.localize(x)
dash = Suppress('-')
colon = Suppress(':')
diff --git a/gracedb/reports.py b/gracedb/reports.py
index effd01e7b..9725cc5e4 100644
--- a/gracedb/reports.py
+++ b/gracedb/reports.py
@@ -24,9 +24,10 @@ import matplotlib.pyplot as plot
import StringIO
import base64
import sys
-import time
-from datetime import datetime, timedelta
+import calendar
+from datetime import timedelta
from utils import posixToGpsTime
+from django.utils import timezone
@internal_user_required
def histo(request):
@@ -76,7 +77,7 @@ def histo(request):
def rate_data():
# XXX there is a better way -- should be using group_by or something.
# WAAY too many queries (~300) going on here.
- now = datetime.now()
+ now = timezone.now()
day = timedelta(1)
ts_min = now - 60 * day
@@ -149,11 +150,12 @@ def cbc_report(request, format=""):
if request.method == "GET":
if "query" not in request.GET:
# Use default query. LowMass events from the past week.
- t_high = datetime.now()
+ t_high = timezone.now()
dt = timedelta(days=7)
t_low = t_high - dt
- t_high = posixToGpsTime(time.mktime(t_high.timetuple()))
- t_low = posixToGpsTime(time.mktime(t_low.timetuple()))
+ # Now the times are in UTC. So we can't use mktime to get posix time.
+ t_high = posixToGpsTime(calendar.timegm(t_high.timetuple()))
+ t_low = posixToGpsTime(calendar.timegm(t_low.timetuple()))
query = 'CBC LowMass %d .. %d' % (t_low, t_high)
rawquery = query
form = SimpleSearchForm({'query': query})
diff --git a/gracedb/templatetags/flash.py b/gracedb/templatetags/flash.py
index 4093b2db2..f0c827d3b 100644
--- a/gracedb/templatetags/flash.py
+++ b/gracedb/templatetags/flash.py
@@ -34,7 +34,8 @@ django.contrib.sessions.middleware.SessionMiddleware
from django import template
from django.template import resolve_variable, Context
-import datetime
+from datetime import timedelta
+from django.utils import timezone
from django.template.loader import render_to_string
from django.contrib.sessions.models import Session
from django.conf import settings
@@ -59,7 +60,7 @@ def session_clear(session):
# Save changes to session
if(session.session_key):
Session.objects.save(session.session_key, session._session,
- datetime.datetime.now() + datetime.timedelta(seconds=settings.SESSION_COOKIE_AGE))
+ timezone.now() + timedelta(seconds=settings.SESSION_COOKIE_AGE))
class RunFlashBlockNode(template.Node):
diff --git a/gracedb/templatetags/timeutil.py b/gracedb/templatetags/timeutil.py
index 3eaee57bd..c2c9248c9 100644
--- a/gracedb/templatetags/timeutil.py
+++ b/gracedb/templatetags/timeutil.py
@@ -49,17 +49,20 @@ def get_multitime_value(t, label, autoescape, format):
dt = t
if not dt.tzinfo:
dt = SERVER_TZ.localize(dt)
- #dt = dt.astimezone(pytz.utc)
+ # XXX in order for mktime to give correct results, the time must be
+ # in the server's timezone.
+ dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
elif isinstance(t, int) or isinstance(t, long):
gps_time = t
dt = gpsToUtc(t)
- posix_time = time.mktime(dt.timetuple())
+ # Note: must convert to server timezone before calling mktime
+ posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
elif isinstance(t, decimal.Decimal):
gps_time = float(t)
dt = gpsToUtc(t)
- posix_time = time.mktime(dt.timetuple())
+ posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
else:
return "N/A"
return '<time utc="%s" gps="%s" llo="%s" lho="%s" virgo="%s" jsparsable="%s"%s>%s</time>' % \
@@ -147,6 +150,8 @@ def gpsdate_tz(gpstime, label="utc"):
def gpstime(dt):
if not dt.tzinfo:
dt = SERVER_TZ.localize(dt)
+ # convert to SERVER_TZ if not already
+ dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
return gps_time
@@ -160,17 +165,17 @@ def timeSelections(t):
dt = t
if not dt.tzinfo:
dt = SERVER_TZ.localize(dt)
- #dt = dt.astimezone(pytz.utc)
+ dt = dt.astimezone(SERVER_TZ)
posix_time = time.mktime(dt.timetuple())
gps_time = int(posixToGpsTime(posix_time))
elif isinstance(t, int) or isinstance(t, long):
gps_time = t
dt = gpsToUtc(t)
- posix_time = time.mktime(dt.timetuple())
+ posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
elif isinstance(t, decimal.Decimal):
gps_time = float(t)
dt = gpsToUtc(t)
- posix_time = time.mktime(dt.timetuple())
+ posix_time = time.mktime(dt.astimezone(SERVER_TZ).timetuple())
else:
raise ValueError("time must be type int, long or datetime, not '%s'" % type(t))
diff --git a/gracedb/view_logic.py b/gracedb/view_logic.py
index 8c6ebfe1d..d529096c4 100644
--- a/gracedb/view_logic.py
+++ b/gracedb/view_logic.py
@@ -31,6 +31,7 @@ import datetime
#import dateutil
from dateutil import parser
import logging
+import pytz
def _createEventFromForm(request, form):
saved = False
@@ -234,9 +235,14 @@ def get_performance_info():
# Now parse the log file
dateformat = '%Y-%m-%dT%H:%M:%S' # ISO format. I think.
- # Lookback time is 3 days.
- dt_now = datetime.datetime.now()
+ # Lookback time is 3 days. These are in UTC.
+ dt_now = timezone.now()
dt_min = dt_now + datetime.timedelta(days=-3)
+
+ # Convert to local time
+ SERVER_TZ = pytz.timezone(settings.TIME_ZONE)
+ dt_now = dt_now.astimezone(SERVER_TZ)
+ dt_min = dt_min.astimezone(SERVER_TZ)
totals_by_status = {}
totals_by_method = {}
@@ -245,6 +251,8 @@ def get_performance_info():
datestring = line[0:len('YYYY-MM-DDTHH:MM:SS')]
# Check the date to see whether it's fresh enough
dt = datetime.datetime.strptime(datestring, dateformat)
+ # Localize so we can compare with aware datetimes
+ dt = SERVER_TZ.localize(dt)
if dt > dt_min:
# Get rid of the datestring and the final colon.
line = line[len(datestring)+1:]
@@ -539,6 +547,8 @@ def create_emobservation(request, event):
try:
start_time = parser.parse(start_time)
+ if not start_time.tzinfo:
+ start_time = pytz.utc.localize(start_time)
except:
raise ValueError('Could not parse start time list element %d of %s'%(i, startTimeRealList))
diff --git a/gracedb/view_utils.py b/gracedb/view_utils.py
index a8ee6d9b0..1b0f0c9b0 100644
--- a/gracedb/view_utils.py
+++ b/gracedb/view_utils.py
@@ -29,8 +29,8 @@ GRACEDB_DATA_DIR = settings.GRACEDB_DATA_DIR
import json
import pytz
-from datetime import datetime
-from time import mktime
+import time
+import calendar
SERVER_TZ = pytz.timezone(settings.TIME_ZONE)
def timeToUTC(dt):
@@ -483,12 +483,13 @@ def skymapViewerEMObservationToDict(emo, request=None):
for t in startTimeList:
time_count += 1
# timetuple throws away the microsecond for some reason
- avg_time_s += mktime(t.timetuple()) + float(t.microsecond)/1e6
+ # Note: the datetimes in the startTimeList are in UTC.
+ avg_time_s += calendar.timegm(t.timetuple()) + float(t.microsecond)/1e6
if time_count > 0:
avg_time_s /= time_count
- avg_time = datetime.fromtimestamp(avg_time_s)
+ avg_time = time.gmtime(avg_time_s)
avg_time_string = avg_time.strftime("%a %b %d %H:%M:%S UTC %Y")
return {
diff --git a/ligoauth/middleware/auth.py b/ligoauth/middleware/auth.py
index 47c524a2e..dcfe60a60 100644
--- a/ligoauth/middleware/auth.py
+++ b/ligoauth/middleware/auth.py
@@ -14,7 +14,7 @@ from django.http import HttpResponse, HttpResponseForbidden
proxyPattern = re.compile(r'^(.*?)(/CN=\d+)*$')
-from datetime import datetime
+from django.utils import timezone
from base64 import b64decode
import json
@@ -153,7 +153,7 @@ class LigoAuthMiddleware:
# actually use 'date_joined' for it's intended purpose.
# check: is now greater than date_joined + time_delta?
if user:
- if datetime.now() > user.date_joined + settings.PASSWORD_EXPIRATION_TIME:
+ if timezone.now() > user.date_joined + settings.PASSWORD_EXPIRATION_TIME:
msg = "Your password has expired. Please log in and request another."
return HttpResponseForbidden(json.dumps({'error': msg}))
diff --git a/settings/default.py b/settings/default.py
index 2bff96c7c..f07e76166 100644
--- a/settings/default.py
+++ b/settings/default.py
@@ -1,5 +1,13 @@
from settings_secret import *
+USE_TZ = True
+
+import warnings
+warnings.filterwarnings(
+ 'error', r"DateTimeField .* received a naive datetime",
+ RuntimeWarning, r'django\.db\.models\.fields')
+
+
# Suitable for production
ALLOWED_HOSTS = ['*']
diff --git a/templates/gracedb/query_help_frag.html b/templates/gracedb/query_help_frag.html
index 756ffc37f..e4ef40965 100644
--- a/templates/gracedb/query_help_frag.html
+++ b/templates/gracedb/query_help_frag.html
@@ -24,7 +24,7 @@
<h4>By Creation Time</h4>
Creation time may be indicated by an exact time or a range. Date/times are
- in the format <code>2009-10-20 13:00:00</code>. If the time is omitted, it
+ in the format <code>2009-10-20 13:00:00</code> (must be UTC). If the time is omitted, it
is assumed to be <code>00:00:00</code>. Dates may also consist of certain
variants of English-like phrases.
The <code>created:</code> keyword is (generally) optional.
diff --git a/userprofile/views.py b/userprofile/views.py
index fa07e9a16..023a41426 100644
--- a/userprofile/views.py
+++ b/userprofile/views.py
@@ -15,7 +15,7 @@ from forms import ContactForm, triggerFormFactory
from gracedb.permission_utils import internal_user_required, lvem_user_required
-from datetime import datetime
+from django.utils import timezone
from gracedb.query import labelQuery
from gracedb.models import Label
@@ -38,7 +38,7 @@ def managePassword(request):
password = User.objects.make_random_password(length=20)
d['password'] = password
request.user.set_password(password)
- request.user.date_joined = datetime.now()
+ request.user.date_joined = timezone.now()
request.user.save()
return render_to_response('profile/manage_password.html',
d,
--
GitLab