docker/backports.pref

+Explanation: shibboleth 3.0 dependencies
+Package: init-system-helpers libxerces-c3.2
+Pin: release a=stretch-backports
+Pin-Priority: 500

docker/check_shibboleth_status

-#!/usr/bin/python
+#!/usr/bin/python3
 
 '''
 Pulls Shibboleth status.sso page, checks for:
@@ -8,8 +8,14 @@ Run ./check_shibboleth_status -h for help.
 '''
 
 # Imports
-import argparse, urllib2, sys
+import argparse
+import sys
 import xml.etree.ElementTree as ET
+try:
+    from urllib.request import urlopen
+    from urllib.error import URLError
+except ImportError:  # python < 3
+    from urllib2 import (urlopen, URLError)
 
 # Parameters - may need to be modified in the future
 # if Shibboleth status pages change or new metadata
@@ -48,12 +54,12 @@ metadata_feeds = args.feeds.split(",")
 # Get XML data from URL.
 host_url = host + "/" + urlpath
 try:
-    response = urllib2.urlopen(host_url, timeout=timeout)
-except urllib2.URLError:
-    print "Error opening Shibboleth status page (" + host_url + ")."
+    response = urlopen(host_url, timeout=timeout)
+except URLError:
+    print("Error opening Shibboleth status page (" + host_url + ").")
     sys.exit(2)
 except:
-    print "Unknown error opening Shibboleth status page (" + host_url + ")."
+    print("Unknown error opening Shibboleth status page (" + host_url + ").")
     sys.exit(3)
 
 # Convert from string to ElementTree
@@ -61,11 +67,11 @@ try:
     status_tree = ET.fromstring(response.read())
 except ET.ParseError:
     # Error parsing response.
-    print "Error parsing response from server - not in XML format."
+    print("Error parsing response from server - not in XML format.")
     sys.exit(2)
 except:
     # Error that is not ParseError.
-    print "Unknown error occurred when parsing response from server."
+    print("Unknown error occurred when parsing response from server.")
     sys.exit(3)
 response.close()
 
@@ -75,12 +81,12 @@ response.close()
 for tag in tags_to_check:
     status_tag = status_tree.find(tag)
     if (status_tag is None):
-        print "Error: tag \'" +  tag + "\' not found."
+        print("Error: tag \'" +  tag + "\' not found.")
         sys.exit(2)
     else:
         status_OK = status_tag.find('OK')
         if (status_OK is None):
-            print "Error: tag \'" + tag + "\' is not OK."
+            print("Error: tag \'" + tag + "\' is not OK.")
             sys.exit(2)
 
 # Check 2: make sure metadata feeds that we expect
@@ -90,12 +96,12 @@ srcs = [element.attrib['source'] for element in metaprov_tags]
 for feed in metadata_feeds:
     feed_found = [src.lower().find(feed) >= 0 for src in srcs]
     if (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + " not found."
+        print("MetadataProvider " + feed + " not found.")
         sys.exit(2)
     elif (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + "found in multiple elements."
+        print("MetadataProvider " + feed + "found in multiple elements.")
         sys.exit(2)
 
 # If we make it to this point, everything is OK.
-print "All MetadataProviders found. Status and SessionCache are OK."
+print("All MetadataProviders found. Status and SessionCache are OK.")
 sys.exit(0)

docker/cleanup

 #!/bin/sh
 
-python /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap
-python /app/gracedb_project/manage.py remove_inactive_alerts
-python /app/gracedb_project/manage.py clearsessions
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap kagra
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap ligo
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap robots
+python3 /app/gracedb_project/manage.py update_catalog_managers_group
+python3 /app/gracedb_project/manage.py remove_inactive_alerts
+python3 /app/gracedb_project/manage.py clearsessions 2>&1 | grep -v segment
+python3 /app/gracedb_project/manage.py remove_old_mdc_public_perms
+PGPASSWORD=$DJANGO_DB_PASSWORD psql -h $DJANGO_DB_HOST -U $DJANGO_DB_USER -c "VACUUM VERBOSE ANALYZE;" $DJANGO_DB_NAME

docker/entrypoint

 #!/bin/bash
 
-export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python  -c 'import uuid; print(uuid.uuid4().hex)')
+export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python3  -c 'import uuid; print(uuid.uuid4().hex)')
+
+# Change the file permissions and ownership on /app/db_data:
+chown gracedb:www-data /app/db_data
+chmod 755 /app/db_data
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/saml_certificate
+SHIB_SP_KEY=/run/secrets/saml_private_key
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
+then
+        echo "Using Shibboleth Cert from docker secrets over the image one"
+        cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+        cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+        chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+        chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+  aws_ses_secret_access_key
+  django_db_password
+  django_secret_key
+  django_twilio_account_sid
+  django_twilio_auth_token
+  lvalert_password
+  igwn_alert_password
+  gracedb_ldap_keytab
+  egad_url
+  egad_api_key
+  django_sentry_dsn"
+
+for SECRET in $LIST
+do
+        VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+        [  -f /run/secrets/$SECRET ] && export $VARNAME="$(< /run/secrets/$SECRET)"
+done
+
+# get x509 cert for ldap access from environment variable. 
+echo "${GRACEDB_LDAP_KEYTAB}" | base64 -d | install -m 0600 /dev/stdin keytab
+kinit ldap/gracedb.ligo.org@LIGO.ORG -k -t keytab
+
 exec "$@"
+

docker/mpm_prefork.conf

+# prefork MPM
+# StartServers: number of server processes to start
+# MinSpareServers: minimum number of server processes which are kept spare
+# MaxSpareServers: maximum number of server processes which are kept spare
+# MaxRequestWorkers: maximum number of server processes allowed to start
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+
+<IfModule mpm_prefork_module>
+        StartServers                     5
+        MinSpareServers           5
+        MaxSpareServers          10
+        MaxRequestWorkers         128
+        ServerLimit               128
+        MaxConnectionsPerChild   0
+</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

docker/shibboleth-ds/idpselect.css

 /* Top level is idpSelectIdPSelector */
 #idpSelectIdPSelector
 {
-    width: 512px;
-    text-align: left;
+    width:fit-content;
+    text-align: center;
     background-color: #FFFFFF;
     border: 2px #A40000 solid;
     padding: 10px;
+    margin-top:25px;
+    margin-bottom:25px;
 }
 
 /* Next down are the idpSelectPreferredIdPTile, idpSelectIdPEntryTile & idpSelectIdPListTile */
@@ -16,7 +18,7 @@
  */
 #idpSelectPreferredIdPTile
 {
-    height:138px; /* Force the height so that the  selector box
+    height: 138px /* Force the height so that the  selector box
                    * goes below when there is only one preslect 
                    */
 }

docker/shibboleth-ds/idpselect_config.js

@@ -22,7 +22,7 @@ function IdPSelectUIParms(){
     this.maxResults = 10;            // How many results to show at once or the number at which to
                                      // start showing if alwaysShow is false
     this.myEntityID = null;          // If non null then this string must match the string provided in the DS parms
-    this.preferredIdP = ['https://login.ligo.org/idp/shibboleth', 'https://login2.ligo.org/idp/shibboleth', 'https://google.cirrusidentity.com/gateway'];
+    this.preferredIdP = ['https://login.ligo.org/idp/shibboleth', 'https://shibbi.pki.itc.u-tokyo.ac.jp/idp/shibboleth', 'https://google.cirrusidentity.com/gateway'];
     this.hiddenIdPs = null;          // Array of entityIds to delete
     this.ignoreKeywords = false;     // Do we ignore the <mdui:Keywords/> when looking for candidates
     this.showListFirst = false;      // Do we start with a list of IdPs or just the dropdown

docker/supervisord-apache2.conf

@@ -6,7 +6,7 @@ redirect_stderr=true
 priority=3
 
 [program:gracedb]
-command=/usr/local/bin/gunicorn config.wsgi:application --reload --config /app/gracedb_project/config/gunicorn_config.py --error-logfile='-' --access-logfile='-'
+command=/usr/local/bin/gunicorn config.wsgi:application --limit-request-field_size 16384 --forwarder-headers 'SCRIPT_NAME,PATH_INFO,REMOTE_USER,ISMEMBEROF,SSL_CLIENT_S_DN,SSL_CLIENT_I_DN,X_FORWARDED_TLS_CLIENT_CERT,X_FORWARDED_TLS_CLIENT_CERT_INFOS' --reload --config /app/gracedb_project/config/gunicorn_config.py --error-logfile='-' --access-logfile='-'
 directory=/app/gracedb_project
 user=gracedb
 group=www-data

docker/supervisord-aws-xray.conf

+[program:aws_xray]
+autostart=%(ENV_ENABLE_AWS_XRAY)s
+command=/usr/bin/xray -f /var/log/xray/xray.log -l info
+user=xray
+group=xray
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100

docker/supervisord-igwn-alert-overseer.conf

+[program:igwn_alert_overseer]
+autostart=%(ENV_ENABLE_IGWN_OVERSEER)s
+command=igwn_alert_overseer -a %(ENV_IGWN_ALERT_USER)s -b %(ENV_IGWN_ALERT_PASSWORD)s
+    -s %(ENV_IGWN_ALERT_SERVER)s -p %(ENV_IGWN_ALERT_OVERSEER_PORT)s
+    -g %(ENV_IGWN_ALERT_GROUP)s 
+    -l - -e - -q - -c -f -i %(ENV_IGWN_ALERT_FLUSH_INTERVAL)s
+user=gracedb
+group=www-data
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100
+

docker/supervisord-lvalert-overseer.conf

-[program:overseer]
-autostart=%(ENV_ENABLE_OVERSEER)s
-command=lvalert_overseer -a %(ENV_LVALERT_USER)s -b %(ENV_LVALERT_PASSWORD)s
-    -s %(ENV_LVALERT_SERVER)s -p %(ENV_LVALERT_OVERSEER_PORT)s
-    -r %(ENV_LVALERT_OVERSEER_RESOURCE)s
-    -l - -e -
-user=gracedb
-group=www-data
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-redirect_stderr=true
-priority=2
-autorestart=true

docker/supervisord-qcluster.conf

+[program:redis_server]
+autostart=%(ENV_DJANGO_ENABLE_LOCAL_REDIS)s
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=3
+autorestart=true
+startretries=100
+
+[program:redis_qcluster]
+autostart=%(ENV_DJANGO_ENABLE_REDIS_QUEUE)s
+command=/usr/bin/python3 /app/gracedb_project/manage.py qcluster --settings %(ENV_DJANGO_SETTINGS_MODULE)s
+user=gracedb
+group=www-data
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100

docs/admin_docs/source/_static/gracedb-nav-style.css

@@ -102,16 +102,17 @@ h1.docnav {
 }
 
 #home #nav-home a,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a,
 #archive #nav-archive a,
 #lab #nav-lab a,
 #reviews #nav-reviews a,
-#userprofile #nav-userprofile a,
 #latest #nav-latest a,
 #contact #nav-contact a {
     background: #a9b0ba;  /* Nav selected color */
@@ -119,16 +120,17 @@ h1.docnav {
     /* text-shadow:none; */
 }
 #home #nav-home a:hover,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a:hover,
 #archive #nav-archive a:hover,
 #lab #nav-lab a:hover,
 #reviews #nav-reviews a:hover,
-#userprofile #nav-userprofile a:hover,
 #latest #nav-latest a:hover,
 #contact #nav-contact a:hover {
     /* background:#e35a00; */

docs/admin_docs/source/_templates/layout.html

@@ -93,8 +93,3 @@ require([
 
 {% endblock %}
 
-{% block header %}
-
-<div id="gracedb-nav-header"></div>
-
-{% endblock %}

docs/admin_docs/source/miscellaneous.rst

@@ -87,12 +87,15 @@ but you still have to go through the same sequence of steps that you would
 for a true developement task. I recommend the workflow described in :ref:`new_server_feature`.
 
 In this particular case, the only necessary code change is to edit the 
-file ``gracedb/events/buildVOEvent.py`` and add something like::
+file ``gracedb/annotations/voevent_utils.py`` and add something like::
 
-    w.add_Param(Param(name="MyParam",
-        dataType="float",
+    p_new = vp.Param(
+        "MyParam",
         value=getMyParamForEvent(event),
-        Description=["My lovely new parameter"]))
+        dataType="float"
+    )
+    p_new.Description = "My lovely new parameter"
+    v.What.append(p_new)
 
 working by analogy with the other parameters present. I only wanted to give
 this example here, because it seems likely that such a task will be considered
@@ -108,7 +111,7 @@ A good starting point is to search the GraceDB server code for "L1" to see where
 Specifics (assume X1 is the IFO code):
 
 1. Add X1OPS, X1OK, X1NO labels, update ``gracedb/templates/gracedb/event_detail_script.js`` with description, and update ``gracedb/templates/search/query_help_frag.html``
-2. Add to instruments in ``gracedb/events/buildVOEvent.py``
+2. Add to instruments in ``gracedb/annotations/voevent_utils.py``
 3. Update ifoList in ``gracedb/events/query.py``
 4. Add entry to ``CONTROL_ROOM_IPS`` in ``gracedb/config/settings/base.py``
 5. Add signoff option for X1 in ``gracedb/templates/gracedb/event_detail.html``

docs/admin_docs/source/public_gracedb.rst

@@ -114,7 +114,7 @@ example, here is how to grant ``view`` permissions to ``public``::
     group_name = 'public'
     perm_shortname = 'view'
 
-    url = g.service_url + urllib.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
+    url = g.service_url + urllib.parse.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
     r = g.put(url)
 
 Templates 

docs/admin_docs/source/robot_certificate.rst

@@ -103,13 +103,14 @@ Edit the migration to do what you want it to do. You could use this as a templat
     ]
 
     def create_robots(apps, schema_editor):
-        RobotUser = apps.get_model('ligoauth', 'RobotUser')
+        User = apps.get_model('auth', 'User')
         X509Cert = apps.get_model('ligoauth', 'X509Cert')
-        Group = apps.get_model('auth', 'Group')
-        lvc_group = Group.objects.get(name=settings.LVC_GROUP)
+        AuthGroup = apps.get_model('ligoauth', 'AuthGroup')
+        lvc_group = AuthGroup.objects.get(name=settings.LVC_GROUP)
+        robot_group = AuthGroup.objects.get(name='robot_accounts')
 
         for entry in ROBOTS:
-            user, created = RobotUser.objects.get_or_create(username=entry['username'])
+            user, created = User.objects.get_or_create(username=entry['username'])
             if created:
                 user.first_name = entry['first_name']
                 user.last_name = entry['last_name']
@@ -121,10 +122,8 @@ Edit the migration to do what you want it to do. You could use this as a templat
 
             # Create the cert objects and link them to our user.
             for dn in entry['dns']:
-                cert, created = X509Cert.objects.get_or_create(subject=dn)
-                if created:
-                    cert.save()
-                cert.users.add(user)
+                cert, created = X509Cert.objects.get_or_create(subject=dn,
+                    user=user)
 
             # Add our user to the LVC group. This permission is required to 
             # do most things, but may *NOT* always be appropriate. It may
@@ -132,14 +131,17 @@ Edit the migration to do what you want it to do. You could use this as a templat
             # a particular pipeline.
             lvc_group.user_set.add(user)
 
+            # Add user to robot accounts
+            robot_group.user_set.add(user)
+
     def delete_robots(apps, schema_editor):
-        RobotUser = apps.get_model('ligoauth', 'RobotUser')
+        User = apps.get_model('auth', 'User')
         X509Cert = apps.get_model('ligoauth', 'X509Cert')
 
         for entry in ROBOTS:
             for dn in entry['dns']:
                 X509Cert.objects.get(subject=dn).delete()
-            RobotUser.objects.get(username=entry['username']).delete()
+            User.objects.get(username=entry['username']).delete()
 
     class Migration(migrations.Migration):
 

docs/admin_docs/source/user_permissions.rst

@@ -62,7 +62,7 @@ in real life (from inside the Django shell)::
     >>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
 
     >>> if p in u.user_permissions.all():
-    ...:    print "Albert can add events!"
+    ...:    print("Albert can add events!")
 
 The Django ``User`` class has a convenience function ``has_perm`` to 
 make this easier::
@@ -72,7 +72,7 @@ make this easier::
     >>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
 
     >>> if u.has_perm('events.add_event'):
-    ...:    print "Albert can add events!"
+    ...:    print("Albert can add events!")
 
 Again, notice that the ``has_perm`` function needs the codename to be scoped by
 the app to which the model belongs. Both are required to fully specify the model.
@@ -124,7 +124,7 @@ event data. Thus, we have added a custom ``view`` permission for the event model
     >>> perms = Permission.objects.filter(codename__startswith='view')
 
     >>> for p in perms:
-    ...:    print p.codename
+    ...:    print(p.codename)
     ...:     
     view_coincinspiralevent
     view_event
@@ -314,7 +314,7 @@ can be done by adding the permission by hand::
     >>> u = User.objects.get(username='albert.einstein@LIGO.ORG')
 
     >>> u.user_permissions.add(p):
-    ...:    print "Albert can add events!"
+    ...:    print("Albert can add events!")
 
 Granting permission to populate a pipeline
 ------------------------------------------

docs/user_docs/source/_static/css/extra.css

+.highlight .err {
+    border: inherit;
+    box-sizing: inherit;
+}

docs/user_docs/source/_static/gracedb-nav-style.css

@@ -102,16 +102,17 @@ h1.docnav {
 }
 
 #home #nav-home a,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a,
 #archive #nav-archive a,
 #lab #nav-lab a,
 #reviews #nav-reviews a,
-#userprofile #nav-userprofile a,
 #latest #nav-latest a,
 #contact #nav-contact a {
     background: #a9b0ba;  /* Nav selected color */
@@ -119,16 +120,17 @@ h1.docnav {
     /* text-shadow:none; */
 }
 #home #nav-home a:hover,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a:hover,
 #archive #nav-archive a:hover,
 #lab #nav-lab a:hover,
 #reviews #nav-reviews a:hover,
-#userprofile #nav-userprofile a:hover,
 #latest #nav-latest a:hover,
 #contact #nav-contact a:hover {
     /* background:#e35a00; */