fixing performance logging

gracedb/view_logic.py

@@ -34,6 +34,7 @@ from dateutil import parser
 from django.utils import timezone
 import logging
 import pytz
+import re
 
 logger = logging.getLogger(__name__)
 
@@ -327,9 +328,6 @@ def get_performance_info():
     logfilepath = settings.LOGGING['handlers']['performance_file']['filename']
     logfile = open(logfilepath, "r")
    
-    # Now parse the log file
-    dateformat = '%Y-%m-%dT%H:%M:%S' # ISO format. I think.
-
     # Lookback time is 3 days. These are in UTC.
     dt_now = timezone.now()
     dt_min = dt_now + datetime.timedelta(days=-3)
@@ -343,20 +341,17 @@ def get_performance_info():
     totals_by_method = {}
 
     for line in logfile:
-        datestring = line[0:len('YYYY-MM-DDTHH:MM:SS')]
+        try:
+            match = re.search(r'^(.*) \| (\w+): (\d+): (\S+)$', line)
+            datestring, method, status, username = match.groups()
+        except:
+            continue
+
         # Check the date to see whether it's fresh enough
-        dt = datetime.datetime.strptime(datestring, dateformat)
+        dt = datetime.datetime.strptime(datestring, settings.LOG_DATEFMT)
         # Localize so we can compare with aware datetimes
-        dt = SERVER_TZ.localize(dt) 
+        dt = SERVER_TZ.localize(dt)
         if dt > dt_min:
-            # Get rid of the datestring and the final colon.
-            line = line[len(datestring)+1:]
-            # Parse
-            method, status, username = line.split(':')
-            method = method.strip()
-            status = int(status.strip())
-            username = username.strip()
-
             if method not in totals_by_method.keys():
                 totals_by_method[method] = 1
                 totals_by_status[method] = {status: 1}
@@ -387,8 +382,8 @@ def get_performance_info():
             'totals_by_status' : totals_by_status,
             'totals_by_method' : totals_by_method,
     }
-    return context
 
+    return context
 
 # 
 # A utility to be used with the gracedb.views.view to determine whether 

settings/default.py

@@ -378,6 +378,7 @@ LOG_FILE_SIZE = 1024*1024 # 1 MB
 LOG_FILE_BAK_CT = 10
 LOG_FORMAT = 'extra_verbose'
 LOG_LEVEL = 'DEBUG'
+LOG_DATEFMT = '%Y-%m-%d %H:%M:%S'
 
 # Note that mode for log files is 'a' (append) by default
 # The 'level' specifier on the handle is optional, and we
@@ -388,16 +389,16 @@ LOGGING = {
     'formatters': {
         'simple': {
             'format': '%(asctime)s | %(message)s',
-            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'datefmt': LOG_DATEFMT,
         },
         'verbose': {
             'format': '%(asctime)s | %(name)s | %(message)s',
-            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'datefmt': LOG_DATEFMT,
         },
         'extra_verbose': {
             'format': '%(asctime)s.%(msecs)03d | %(name)s | %(levelname)s | ' \
                       + '%(filename)s, line %(lineno)s | %(message)s',
-            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'datefmt': LOG_DATEFMT,
         }
     },
     'handlers': {
@@ -422,7 +423,9 @@ LOGGING = {
             'level': 'ERROR',
         },
         'performance_file': {
-            'class': 'logging.FileHandler',
+            'class': 'logging.handlers.ConcurrentRotatingFileHandler',
+            'maxBytes': 1024*1024,
+            'backupCount': 1,
             'formatter': 'simple',
             'filename': '%s/gracedb_performance.log' % LOG_ROOT,
         },