Add a number of management commands

Move a number of scripts which were managed in a separate repo
into management commands.  This way they can be more easily executed
either manually or by cron jobs.

gracedb/events/management/commands/clean_up_performance_log.py

+import datetime
+import re
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+# Parameters
+LOOKBACK_TIME = 5 # days
+LOG_FILE_PATH = settings.LOGGING['handlers']['performance_file']['filename']
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **kwargs):
+
+        # Read log
+        logfile = open(LOG_FILE_PATH, "r")
+        lines = logfile.readlines()
+        logfile.close()
+
+        # Lookback time is 5 days.
+        dt_now = datetime.datetime.now()
+        dt_min = dt_now + datetime.timedelta(days=-1*LOOKBACK_TIME)
+
+        # Get "fresh" enough log messages from logfile
+        dateformat = settings.LOG_DATEFMT
+        logfile_str = ""
+        for line in lines:
+
+            # Check the date to see whether it's fresh enough
+            match = re.search(r'^(.*) \| .*', line)
+            datestring = match.group(1)
+            dt = datetime.datetime.strptime(datestring, dateformat)
+            if dt > dt_min:
+                logfile_str += line
+
+        # Overwrite file
+        logfile = open(LOG_FILE_PATH, "w")
+        logfile.write(logfile_str)
+        logfile.close()

gracedb/events/management/commands/make_latency_histograms.py

+from datetime import timedelta
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as pyplot
+import numpy
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from events.models import Event, Pipeline
+
+DEST_DIR = settings.LATENCY_REPORT_DEST_DIR
+MAX_X = settings.LATENCY_MAXIMUM_CHARTED
+WEB_PAGE_FILE_PATH = settings.LATENCY_REPORT_WEB_PAGE_FILE_PATH
+URL_PREFIX = settings.REPORT_INFO_URL_PREFIX
+# XXX Branson introduced during ER6 to clean things up a bit.
+PIPELINE_EXCLUDE_LIST = ['HardwareInjection', 'X', 'Q', 'Omega', 'Ringdown',
+    'LIB', 'SNEWS', 'pycbc', 'CWB2G']
+
+
+def writeIndex(notes, fname):
+
+    createdDate = str(timezone.now())
+    maxx = MAX_X
+
+    table = '<table border="1" bgcolor="white">'
+    table += """<caption>Tables generated: %s<br/>
+                      Maximum charted latency: %s seconds</caption>""" \
+             % (createdDate, maxx)
+    table += "<tr><th>&nbsp;</th>"
+    for time_range in ['day', 'week', 'month']:
+        table += "<th>last %s</th>" % time_range
+    table += "</tr>"
+    for pipeline in Pipeline.objects.all():
+    #for atype, atype_name in Event.ANALYSIS_TYPE_CHOICES:
+        if pipeline.name in PIPELINE_EXCLUDE_LIST:
+            continue
+        pname = pipeline.name
+        table += "<tr>"
+        table += "<td>%s</td>" % pname
+        for time_range in ['day', 'week', 'month']:
+            table += '<td align="center" bgcolor="white">'
+            n = notes[pname][time_range]
+            extra = ""
+            if n['fname'] is not None:
+                table += '<img width="400" height="300" src="%s"/>' % \
+                           (URL_PREFIX + os.path.basename(n['fname']))
+                extra = "%d total events" % n['count']
+            else:
+                extra = "No Applicable Events"
+            if n['over'] != 0:
+                extra += "<br/>%d events over maximum latency of %s seconds" % (n['over'], MAX_X)
+            table += "<br/>%s" % extra
+            table += "</td>"
+        table += "</tr>"
+    table += "</table>"
+
+    f = open(fname, "w")
+    f.write(table)
+    f.close()
+
+
+def makePlot(data, title, maxx=1800, facecolor='green'):
+    # convert data to float (might be Decimal type)
+    data = [float(d) for d in data]
+
+    # make sure plot is clear!
+    pyplot.close()
+    #nbins = maxx / 30
+    nbins = numpy.logspace(1.3, numpy.log10(maxx), 50)
+
+    pyplot.xlim([20,maxx])
+    fig = pyplot.figure()
+
+    ax = fig.add_axes((.1, .1, .8, .8))
+
+    n, bins, patches = ax.hist(data, nbins, facecolor=facecolor)
+
+    vmax = max(n)
+    if vmax <= 10:
+        vmax = 10
+    elif (vmax%10) == 0:
+        vmax += 10
+    else:
+        vmax += 10 - (vmax % 10)
+
+    ax.set_xlabel('Seconds', fontsize=20)
+    ax.set_ylabel('Number of Events', fontsize=20)
+    ax.set_xscale('log')
+    ax.axis([20, maxx, 0, vmax])
+    ax.grid(True)
+
+    return pyplot
+
+
+class Command(BaseCommand):
+    help="Create latency histograms"
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        now = timezone.now()
+
+        start_day = now - timedelta(1)
+        start_week = now - timedelta(7)
+        start_month = now - timedelta(30)
+
+        time_ranges =  [(start_day, "day"), (start_week, "week"), (start_month,
+            "month")]
+
+        annotations = {}
+
+        # Make the histograms, save as png's.
+        for pipeline in Pipeline.objects.all():
+            if pipeline.name in PIPELINE_EXCLUDE_LIST:
+                continue
+            pname = pipeline.name
+            annotations[pname] = {}
+            for start_time, time_range in time_ranges:
+                note = {}
+                fname = os.path.join(DEST_DIR, "%s-%s.png" % (pname, time_range))
+                note['fname'] = fname
+                data = Event.objects.filter(pipeline=pipeline,
+                                            created__range=[start_time, now],
+                                            gpstime__gt=0) \
+                                    .exclude(group__name="Test")
+                note['count'] = data.count()
+                data = [e.reportingLatency() for e in data]
+                data = [d for d in data if d <= MAX_X and d > 0]
+                note['npoints'] = len(data)
+                note['over'] = note['count'] - note['npoints']
+                if note['npoints'] <= 0:
+                    try:
+                        note['fname'] = None
+                        os.unlink(fname)
+                    except OSError:
+                        pass
+                else:
+                    makePlot(data, pname, maxx=MAX_X).savefig(fname)
+                annotations[pname][time_range] = note
+
+        writeIndex(annotations, WEB_PAGE_FILE_PATH)
+

gracedb/events/management/commands/write_binned_counts.py

+from datetime import timedelta, datetime
+from dateutil import parser
+import json
+import pytz
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from events.models import Event, Pipeline, Search, Group
+
+# Default settings
+LOOKBACK_HOURS = 720
+BIN_WIDTH = 24
+
+# Set up time range
+end_time = datetime.utcnow()
+end_time = end_time.replace(hour=0, minute=0, second=0, microsecond=0)
+end_time = pytz.utc.localize(end_time)
+start_time = end_time - timedelta(hours=LOOKBACK_HOURS)
+# Convert to ISOformat
+start_time = start_time.isoformat()
+end_time = end_time.isoformat()
+
+# get_counts_for_bin
+# Takes as input:
+# - the lower bin boundary (a naive datetime object in UTC)
+# - the bin_width in hours
+# - the pipeline we are interested in
+# Returns the number of events in that bin, excluding MDC and Test.
+MDC = Search.objects.get(name='MDC')
+Test = Group.objects.get(name='Test')
+
+# make a list of pipeline objects
+PIPELINES = []
+for n in settings.BINNED_COUNT_PIPELINES:
+    try:
+        PIPELINES.append(Pipeline.objects.get(name=n))
+    except:
+        pass
+OTHER_PIPELINES = []
+for p in Pipeline.objects.all():
+    if p.name not in PIPELINES:
+        OTHER_PIPELINES.append(p)
+
+
+def get_counts_for_bin(lbb, bin_width, pipeline):
+    ubb = lbb + timedelta(hours=bin_width)
+    events = Event.objects.filter(pipeline=pipeline, created__range=(lbb, ubb))
+    if MDC:
+        events = events.exclude(search=MDC)
+    if Test:
+        events = events.exclude(group=Test)
+    return events.count()
+
+
+# given a date string, parse it and localize to UTC if necessary
+def parse_and_localize(date_string):
+    if not date_string:
+        return None
+    dt = parser.parse(date_string)
+    if not dt.tzinfo:
+        dt = pytz.utc.localize(dt)
+    return dt
+
+
+def get_record(lbb, bin_width):
+    bc = lbb + timedelta(hours=bin_width/2)
+    r = { 'time': bc, 'delta_t': bin_width }
+    total = 0
+    for p in PIPELINES:
+        count = get_counts_for_bin(lbb, bin_width, p)
+        total += count
+        r[p.name] = count
+    other = 0
+    for p in OTHER_PIPELINES:
+        other += get_counts_for_bin(lbb, bin_width, p)
+    r['Other'] = other
+    total += other
+    r['Total'] = total
+    return r
+
+
+def dt_record(r):
+    r['time'] = parse_and_localize(r['time'])
+    return r
+
+
+def strtime_record(r):
+    r['time'] = r['time'].replace(tzinfo=None).isoformat()
+    return r
+
+
+class Command(BaseCommand):
+    help = 'Manage the binned counts file used for plotting rates.'
+
+    def handle(self, *args, **options):
+        # First of all, that bin width had better be an even number of hours.
+        bin_width = BIN_WIDTH
+        if bin_width % 2 != 0:
+            raise ValueError("Bin width must be divisible by 2. Sorry.")
+
+        # Let's take our desired range and turn it into UTC datetime objects
+        start = parse_and_localize(start_time)
+        end = parse_and_localize(end_time)
+
+        duration = end - start
+        # This timedelta has days, seconds, and total seconds.
+        # What we want to verify is that that is an integer number of hours.
+        # That is, the total seconds should be divisible by 3600.
+        hours, r_seconds = divmod(duration.total_seconds(), 3600)
+        if r_seconds != 0.0:
+            msg = "The start and end times must be separated by an integer number of hours."
+            raise ValueError(msg)
+
+        # Now we need to verify that the number of hours is divisible by our
+        # bin width
+        bins, r_hours = divmod(hours, bin_width)
+        bins = int(bins)
+
+        if r_hours != 0.0:
+            msg = "The start and end times must correspond to an integer number of bins."
+            raise ValueError(msg)
+
+        # read in the file and interpret it as JSON
+        f = None
+        try:
+            f = open(settings.BINNED_COUNT_FILE, 'r')
+        except:
+            pass
+
+        records = []
+        if f:
+            try:
+                records = json.loads(f.read())
+            except:
+                pass
+            f.close()
+
+        # process the records so that the time is a datetime for all of them
+        # Note that the times here are at the bin centers
+        records = [dt_record(r) for r in records]
+
+        # accumlate the necessary records
+        new_records = []
+        for i in range(bins):
+            lbb = start + timedelta(hours = i*bin_width)
+            bc = lbb + timedelta(hours = bin_width/2)
+            # look for an existing record with the desired lower bin
+            # boundary and delta.
+            found = False
+            for r in records:
+                if bc == r['time'] and bin_width == r['delta_t']:
+                    found = True
+                    new_records.append(r)
+            if not found:
+                new_records.append(get_record(lbb, bin_width))
+
+        new_records = [strtime_record(r) for r in new_records]
+
+        # write out the file
+        f = open(settings.BINNED_COUNT_FILE, 'w')
+        f.write(json.dumps(new_records))
+        f.close()

gracedb/ligoauth/management/commands/update_user_accounts_from_ligo_ldap.py

+from django.core.management.base import BaseCommand, CommandError
+
+import datetime
+from ligoauth.models import LigoLdapUser, X509Cert, AlternateEmail
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+import ldap
+
+# Variables for LDAP search
+BASE_DN = "ou=people,dc=ligo,dc=org"
+SEARCH_SCOPE = ldap.SCOPE_SUBTREE
+SEARCH_FILTER = "(employeeNumber=*)"
+RETRIEVE_ATTRIBUTES = [
+    "krbPrincipalName",
+    "gridX509subject",
+    "givenName",
+    "sn",
+    "mail",
+    "isMemberOf",
+    "mailAlternateAddress",
+    "mailForwardingAddress"
+]
+LDAP_ADDRESS = "ldap.ligo.org"
+LDAP_PORT = 636
+
+class Command(BaseCommand):
+    help="Get updated user data from LIGO LDAP"
+
+    def add_arguments(self, parser):
+        parser.add_argument('-q', '--quiet', action='store_true',
+            default=False, help='Suppress output')
+
+    def handle(self, *args, **options):
+        verbose = not options['quiet']
+        if verbose:
+            self.stdout.write('Refreshing users from LIGO LDAP at {0}' \
+                .format(datetime.datetime.utcnow()))
+
+        # Get LVC group
+        lvc_group = Group.objects.get(name=settings.LVC_GROUP)
+
+        # Open connection to LDAP and run a search
+        l = ldap.initialize("ldaps://{address}:{port}".format(
+            address=LDAP_ADDRESS, port=LDAP_PORT))
+        l.protocol_version = ldap.VERSION3
+        ldap_result_id = l.search(BASE_DN, SEARCH_SCOPE, SEARCH_FILTER,
+            RETRIEVE_ATTRIBUTES)
+
+        # Get all results
+        result_data = True
+        while result_data:
+            result_type, result_data = l.result(ldap_result_id, 0)
+
+            if result_type == ldap.RES_SEARCH_ENTRY:
+                for (ldap_dn, ldap_result) in result_data:
+                    first_name = unicode(ldap_result['givenName'][0], 'utf-8')
+                    last_name = unicode(ldap_result['sn'][0], 'utf-8')
+                    email = ldap_result['mail'][0]
+                    new_dns = set(ldap_result.get('gridX509subject', []))
+                    memberships = ldap_result.get('isMemberOf', [])
+                    is_active = lvc_group.name in memberships
+                    principal = ldap_result['krbPrincipalName'][0]
+
+                    # Update/Create LigoLdapUser entry
+                    defaults = {
+                        'first_name': first_name,
+                        'last_name': last_name,
+                        'email': email,
+                        'username': principal,
+                        'is_active': is_active,
+                    }
+
+                    # Determine if base user and ligoldapuser objects exist
+                    user_exists = User.objects.filter(username=
+                        defaults['username']).exists()
+                    l_user_exists = LigoLdapUser.objects.filter(
+                        ldap_dn=ldap_dn).exists()
+
+                    # Handle different cases
+                    created = False
+                    if l_user_exists:
+                        l_user = LigoLdapUser.objects.get(ldap_dn=ldap_dn)
+                        user = l_user.user_ptr
+                    else:
+                        if user_exists:
+                            user = User.objects.get(username=
+                                defaults['username'])
+                            l_user = LigoLdapUser.objects.create(
+                                ldap_dn=ldap_dn, user_ptr=user)
+                            l_user.__dict__.update(user.__dict__)
+                            if verbose:
+                                self.stdout.write(("Created ligoldapuser "
+                                    "for {0}").format(user.username))
+                        else:
+                            l_user = LigoLdapUser.objects.create(
+                                ldap_dn=ldap_dn, **defaults)
+                            user = l_user.user_ptr
+                            if verbose:
+                                self.stdout.write(("Created user and "
+                                    "ligoldapuser for {0}").format(
+                                    l_user.username))
+                            created = True
+
+                    # Typically a case where the person's username was changed
+                    # and there are now two user accounts in GraceDB
+                    if user.username != defaults['username'] and user_exists:
+                        self.stdout.write(('ERROR: requires manual '
+                            'investigation. LDAP username: {0}, '
+                            'ligoldapuser.user_ptr.username: {1}').format(
+                            defaults['username'], l_user.user_ptr.username))
+                        continue
+
+                    # Update user attributes from LDAP
+                    changed = False
+                    if not created:
+                        for k in defaults:
+                            if (defaults[k] != getattr(user, k)):
+                                setattr(l_user, k, defaults[k])
+                                changed = True
+                    if changed and verbose:
+                        self.stdout.write("User {0} updated".format(
+                            l_user.username))
+
+                    # Revoke staff/superuser if not active.
+                    if ((l_user.is_staff or l_user.is_superuser)
+                        and not is_active):
+                        l_user.is_staff = l_user.is_superuser = False
+                        changed = True
+    
+                    # Try to save user.
+                    if created or changed:
+                        try:
+                            l_user.save()
+                        except Exception as e:
+                            self.stdout.write(("Failed to save user '{0}': "
+                                "{1}.").format(l_user.username, e))
+                            continue
+
+                    # Update X509 certs for user
+                    current_dns = set([c.subject for c in
+                        user.x509cert_set.all()])
+                    if current_dns != new_dns:
+                        for dn in new_dns - current_dns:
+                            cert, created = X509Cert.objects.get_or_create(
+                                subject=dn)
+                            cert.users.add(l_user)
+
+                    # Update group information - we do this only for groups
+                    # that already exist in GraceDB
+                    for g in Group.objects.all():
+                        # Add the user to the group if they aren't a member
+                        if (g.name in memberships and g not in
+                            l_user.groups.all()):
+
+                            g.user_set.add(l_user)
+                            if verbose:
+                                self.stdout.write("Adding {0} to {1}".format(
+                                    l_user.username, g.name))
+
+                    # Remove the user from the LVC group if they are no longer
+                    # a member. This is the only group in GraceDB which is
+                    # populated from the LIGO LDAP.
+                    if (lvc_group.name not in memberships and
+                        lvc_group in l_user.groups.all()):
+
+                        l_user.groups.remove(lvc_group)
+                        if verbose:
+                            self.stdout.write("Removing {user} from {group}" \
+                                .format(user=l_user.username,
+                                group=lvc_group.name))
+
+                    # Get alternate email addresses (for some reason...)
+                    try:
+                        mailForwardingAddress = unicode(ldap_result['mailForwardingAddress'][0])
+                    except:
+                        mailForwardingAddress = None
+                    mailAlternateAddresses = ldap_result.get('mailAlternateAddress', [])
+
+                    # Finally, deal with alternate emails.
+                    if mailForwardingAddress:
+                        try:
+                            AlternateEmail.objects.get_or_create(l_user=user,
+                                email=mailForwardingAddress)
+                        except:
+                            pass
+
+                    if len(mailAlternateAddresses) > 0:
+                        for email in mailAlternateAddresses:
+                            try:
+                                AlternateEmail.objects.get_or_create(
+                                    l_user=user, email=email)
+                            except:
+                                pass

gracedb/userprofile/management/commands/remove_inactive_alerts.py

+import datetime
+
+from django.conf import settings
+from django.contrib.auth.models import Group
+from django.core.management.base import BaseCommand, CommandError
+
+from userprofile.models import Contact, Trigger
+
+
+class Command(BaseCommand):
+    help="Delete Contacts and Notifications for inactive users"
+
+    def add_arguments(self, parser):
+        parser.add_argument('-q', '--quiet', action='store_true',
+            default=False, help='Suppress output')
+
+    def handle(self, *args, **options):
+        verbose = not options['quiet']
+
+        if verbose:
+            self.stdout.write(('Checking inactive users\' triggers and '
+                'contacts at {0}').format(datetime.datetime.utcnow()))
+
+        # Get contacts and triggers whose user is no longer in the LVC
+        lvc = Group.objects.get(name=settings.LVC_GROUP)
+        triggers = Trigger.objects.exclude(user__groups=lvc)
+        contacts = Contact.objects.exclude(user__groups=lvc)
+
+        # Generate log message
+        if verbose:
+            if triggers.exists():
+                t_log_msg = "Deleting {0} triggers: ".format(triggers.count())\
+                    + " | ".join([t.__str__() for t in triggers])
+                self.stdout.write(t_log_msg)
+            if contacts.exists():
+                c_log_msg = "Deleting {0} contacts: ".format(contacts.count())\
+                     + " | ".join([c.__str__() for c in contacts])
+                self.stdout.write(c_log_msg)
+
+        # Delete
+        triggers.delete()
+        contacts.delete()