docker/backports.pref

+Explanation: shibboleth 3.0 dependencies
+Package: init-system-helpers libxerces-c3.2
+Pin: release a=stretch-backports
+Pin-Priority: 500

docker/check_shibboleth_status

-#!/usr/bin/python
+#!/usr/bin/python3
 
 '''
 Pulls Shibboleth status.sso page, checks for:
@@ -8,8 +8,14 @@ Run ./check_shibboleth_status -h for help.
 '''
 
 # Imports
-import argparse, urllib2, sys
+import argparse
+import sys
 import xml.etree.ElementTree as ET
+try:
+    from urllib.request import urlopen
+    from urllib.error import URLError
+except ImportError:  # python < 3
+    from urllib2 import (urlopen, URLError)
 
 # Parameters - may need to be modified in the future
 # if Shibboleth status pages change or new metadata
@@ -48,12 +54,12 @@ metadata_feeds = args.feeds.split(",")
 # Get XML data from URL.
 host_url = host + "/" + urlpath
 try:
-    response = urllib2.urlopen(host_url, timeout=timeout)
-except urllib2.URLError:
-    print "Error opening Shibboleth status page (" + host_url + ")."
+    response = urlopen(host_url, timeout=timeout)
+except URLError:
+    print("Error opening Shibboleth status page (" + host_url + ").")
     sys.exit(2)
 except:
-    print "Unknown error opening Shibboleth status page (" + host_url + ")."
+    print("Unknown error opening Shibboleth status page (" + host_url + ").")
     sys.exit(3)
 
 # Convert from string to ElementTree
@@ -61,11 +67,11 @@ try:
     status_tree = ET.fromstring(response.read())
 except ET.ParseError:
     # Error parsing response.
-    print "Error parsing response from server - not in XML format."
+    print("Error parsing response from server - not in XML format.")
     sys.exit(2)
 except:
     # Error that is not ParseError.
-    print "Unknown error occurred when parsing response from server."
+    print("Unknown error occurred when parsing response from server.")
     sys.exit(3)
 response.close()
 
@@ -75,12 +81,12 @@ response.close()
 for tag in tags_to_check:
     status_tag = status_tree.find(tag)
     if (status_tag is None):
-        print "Error: tag \'" +  tag + "\' not found."
+        print("Error: tag \'" +  tag + "\' not found.")
         sys.exit(2)
     else:
         status_OK = status_tag.find('OK')
         if (status_OK is None):
-            print "Error: tag \'" + tag + "\' is not OK."
+            print("Error: tag \'" + tag + "\' is not OK.")
             sys.exit(2)
 
 # Check 2: make sure metadata feeds that we expect
@@ -90,12 +96,12 @@ srcs = [element.attrib['source'] for element in metaprov_tags]
 for feed in metadata_feeds:
     feed_found = [src.lower().find(feed) >= 0 for src in srcs]
     if (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + " not found."
+        print("MetadataProvider " + feed + " not found.")
         sys.exit(2)
     elif (sum(feed_found) < 1):
-        print "MetadataProvider " + feed + "found in multiple elements."
+        print("MetadataProvider " + feed + "found in multiple elements.")
         sys.exit(2)
 
 # If we make it to this point, everything is OK.
-print "All MetadataProviders found. Status and SessionCache are OK."
+print("All MetadataProviders found. Status and SessionCache are OK.")
 sys.exit(0)

docker/cleanup

 #!/bin/sh
 
-python /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap
-python /app/gracedb_project/manage.py remove_inactive_alerts
-python /app/gracedb_project/manage.py clearsessions
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap kagra
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap ligo
+python3 /app/gracedb_project/manage.py update_user_accounts_from_ligo_ldap robots
+python3 /app/gracedb_project/manage.py update_catalog_managers_group
+python3 /app/gracedb_project/manage.py remove_inactive_alerts
+python3 /app/gracedb_project/manage.py clearsessions 2>&1 | grep -v segment
+python3 /app/gracedb_project/manage.py remove_old_mdc_public_perms
+PGPASSWORD=$DJANGO_DB_PASSWORD psql -h $DJANGO_DB_HOST -U $DJANGO_DB_USER -c "VACUUM VERBOSE ANALYZE;" $DJANGO_DB_NAME

docker/entrypoint

 #!/bin/bash
 
-export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python  -c 'import uuid; print(uuid.uuid4().hex)')
+export LVALERT_OVERSEER_RESOURCE=${LVALERT_USER}_overseer_$(python3  -c 'import uuid; print(uuid.uuid4().hex)')
+
+# Change the file permissions and ownership on /app/db_data:
+chown gracedb:www-data /app/db_data
+chmod 755 /app/db_data
+
+## PGA: 2019-10-15: use certs from secrets for Shibboleth SP
+SHIB_SP_CERT=/run/secrets/saml_certificate
+SHIB_SP_KEY=/run/secrets/saml_private_key
+if [[ -f $SHIB_SP_CERT && -f $SHIB_SP_KEY ]]
+then
+        echo "Using Shibboleth Cert from docker secrets over the image one"
+        cp -f $SHIB_SP_CERT /etc/shibboleth/sp-cert.pem
+        cp -f $SHIB_SP_KEY /etc/shibboleth/sp-key.pem
+        chown _shibd:_shibd /etc/shibboleth/sp-{cert,key}.pem
+        chmod 0600 /etc/shibboleth/sp-key.pem
+fi
+
+## PGA 2019-10-16: use secrets for sensitive environment variables
+LIST="aws_ses_access_key_id
+  aws_ses_secret_access_key
+  django_db_password
+  django_secret_key
+  django_twilio_account_sid
+  django_twilio_auth_token
+  lvalert_password
+  igwn_alert_password
+  gracedb_ldap_keytab
+  egad_url
+  egad_api_key
+  django_sentry_dsn"
+
+for SECRET in $LIST
+do
+        VARNAME=$( tr [:lower:] [:upper:] <<<$SECRET)
+        [  -f /run/secrets/$SECRET ] && export $VARNAME="$(< /run/secrets/$SECRET)"
+done
+
+# get x509 cert for ldap access from environment variable. 
+echo "${GRACEDB_LDAP_KEYTAB}" | base64 -d | install -m 0600 /dev/stdin keytab
+kinit ldap/gracedb.ligo.org@LIGO.ORG -k -t keytab
+
 exec "$@"
+

docker/mpm_prefork.conf

+# prefork MPM
+# StartServers: number of server processes to start
+# MinSpareServers: minimum number of server processes which are kept spare
+# MaxSpareServers: maximum number of server processes which are kept spare
+# MaxRequestWorkers: maximum number of server processes allowed to start
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+
+<IfModule mpm_prefork_module>
+        StartServers                     5
+        MinSpareServers           5
+        MaxSpareServers          10
+        MaxRequestWorkers         128
+        ServerLimit               128
+        MaxConnectionsPerChild   0
+</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

docker/shibboleth-ds/idpselect.css

 /* Top level is idpSelectIdPSelector */
 #idpSelectIdPSelector
 {
-    width: 512px;
-    text-align: left;
+    width:fit-content;
+    text-align: center;
     background-color: #FFFFFF;
     border: 2px #A40000 solid;
     padding: 10px;
+    margin-top:25px;
+    margin-bottom:25px;
 }
 
 /* Next down are the idpSelectPreferredIdPTile, idpSelectIdPEntryTile & idpSelectIdPListTile */
@@ -16,7 +18,7 @@
  */
 #idpSelectPreferredIdPTile
 {
-    height:138px; /* Force the height so that the  selector box
+    height: 138px /* Force the height so that the  selector box
                    * goes below when there is only one preslect 
                    */
 }

docker/shibboleth-ds/idpselect_config.js

@@ -22,7 +22,7 @@ function IdPSelectUIParms(){
     this.maxResults = 10;            // How many results to show at once or the number at which to
                                      // start showing if alwaysShow is false
     this.myEntityID = null;          // If non null then this string must match the string provided in the DS parms
-    this.preferredIdP = ['https://login.ligo.org/idp/shibboleth', 'https://login2.ligo.org/idp/shibboleth', 'https://google.cirrusidentity.com/gateway'];
+    this.preferredIdP = ['https://login.ligo.org/idp/shibboleth', 'https://shibbi.pki.itc.u-tokyo.ac.jp/idp/shibboleth', 'https://google.cirrusidentity.com/gateway'];
     this.hiddenIdPs = null;          // Array of entityIds to delete
     this.ignoreKeywords = false;     // Do we ignore the <mdui:Keywords/> when looking for candidates
     this.showListFirst = false;      // Do we start with a list of IdPs or just the dropdown

docker/supervisord-apache2.conf

@@ -6,7 +6,7 @@ redirect_stderr=true
 priority=3
 
 [program:gracedb]
-command=/usr/local/bin/gunicorn config.wsgi:application --reload --config /app/gracedb_project/config/gunicorn_config.py --error-logfile='-' --access-logfile='-'
+command=/usr/local/bin/gunicorn config.wsgi:application --limit-request-field_size 16384 --forwarder-headers 'SCRIPT_NAME,PATH_INFO,REMOTE_USER,ISMEMBEROF,SSL_CLIENT_S_DN,SSL_CLIENT_I_DN,X_FORWARDED_TLS_CLIENT_CERT,X_FORWARDED_TLS_CLIENT_CERT_INFOS' --reload --config /app/gracedb_project/config/gunicorn_config.py --error-logfile='-' --access-logfile='-'
 directory=/app/gracedb_project
 user=gracedb
 group=www-data

docker/supervisord-aws-xray.conf

+[program:aws_xray]
+autostart=%(ENV_ENABLE_AWS_XRAY)s
+command=/usr/bin/xray -f /var/log/xray/xray.log -l info
+user=xray
+group=xray
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100

docker/supervisord-igwn-alert-overseer.conf

+[program:igwn_alert_overseer]
+autostart=%(ENV_ENABLE_IGWN_OVERSEER)s
+command=igwn_alert_overseer -a %(ENV_IGWN_ALERT_USER)s -b %(ENV_IGWN_ALERT_PASSWORD)s
+    -s %(ENV_IGWN_ALERT_SERVER)s -p %(ENV_IGWN_ALERT_OVERSEER_PORT)s
+    -g %(ENV_IGWN_ALERT_GROUP)s 
+    -l - -e - -q - -c -f -i %(ENV_IGWN_ALERT_FLUSH_INTERVAL)s
+user=gracedb
+group=www-data
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100
+

docker/supervisord-lvalert-overseer.conf

-[program:overseer]
-autostart=%(ENV_ENABLE_OVERSEER)s
-command=lvalert_overseer -a %(ENV_LVALERT_USER)s -b %(ENV_LVALERT_PASSWORD)s
-    -s %(ENV_LVALERT_SERVER)s -p %(ENV_LVALERT_OVERSEER_PORT)s
-    -r %(ENV_LVALERT_OVERSEER_RESOURCE)s
-    -l - -e -
-user=gracedb
-group=www-data
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-redirect_stderr=true
-priority=2
-autorestart=unexpected

docker/supervisord-qcluster.conf

+[program:redis_server]
+autostart=%(ENV_DJANGO_ENABLE_LOCAL_REDIS)s
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=3
+autorestart=true
+startretries=100
+
+[program:redis_qcluster]
+autostart=%(ENV_DJANGO_ENABLE_REDIS_QUEUE)s
+command=/usr/bin/python3 /app/gracedb_project/manage.py qcluster --settings %(ENV_DJANGO_SETTINGS_MODULE)s
+user=gracedb
+group=www-data
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+priority=2
+autorestart=true
+startretries=100

docs/admin_docs/source/_static/gracedb-nav-style.css

@@ -102,16 +102,17 @@ h1.docnav {
 }
 
 #home #nav-home a,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a,
 #archive #nav-archive a,
 #lab #nav-lab a,
 #reviews #nav-reviews a,
-#userprofile #nav-userprofile a,
 #latest #nav-latest a,
 #contact #nav-contact a {
     background: #a9b0ba;  /* Nav selected color */
@@ -119,16 +120,17 @@ h1.docnav {
     /* text-shadow:none; */
 }
 #home #nav-home a:hover,
-#create #nav-create a,
+#public #nav-public a,
 #search #nav-search a,
+#pipelines #nav-pipelines a,
+#alerts #nav-alerts a,
+#password #nav-password a,
 #doc #nav-doc a,
-#reports #nav-reports a,
-#feeds #nav-feeds a,
+#other #nav-other a,
 #about #nav-about a:hover,
 #archive #nav-archive a:hover,
 #lab #nav-lab a:hover,
 #reviews #nav-reviews a:hover,
-#userprofile #nav-userprofile a:hover,
 #latest #nav-latest a:hover,
 #contact #nav-contact a:hover {
     /* background:#e35a00; */

docs/admin_docs/source/_templates/layout.html

@@ -93,8 +93,3 @@ require([
 
 {% endblock %}
 
-{% block header %}
-
-<div id="gracedb-nav-header"></div>
-
-{% endblock %}

docs/admin_docs/source/miscellaneous.rst

@@ -87,12 +87,15 @@ but you still have to go through the same sequence of steps that you would
 for a true developement task. I recommend the workflow described in :ref:`new_server_feature`.
 
 In this particular case, the only necessary code change is to edit the 
-file ``gracedb/events/buildVOEvent.py`` and add something like::
+file ``gracedb/annotations/voevent_utils.py`` and add something like::
 
-    w.add_Param(Param(name="MyParam",
-        dataType="float",
+    p_new = vp.Param(
+        "MyParam",
         value=getMyParamForEvent(event),
-        Description=["My lovely new parameter"]))
+        dataType="float"
+    )
+    p_new.Description = "My lovely new parameter"
+    v.What.append(p_new)
 
 working by analogy with the other parameters present. I only wanted to give
 this example here, because it seems likely that such a task will be considered
@@ -108,7 +111,7 @@ A good starting point is to search the GraceDB server code for "L1" to see where
 Specifics (assume X1 is the IFO code):
 
 1. Add X1OPS, X1OK, X1NO labels, update ``gracedb/templates/gracedb/event_detail_script.js`` with description, and update ``gracedb/templates/search/query_help_frag.html``
-2. Add to instruments in ``gracedb/events/buildVOEvent.py``
+2. Add to instruments in ``gracedb/annotations/voevent_utils.py``
 3. Update ifoList in ``gracedb/events/query.py``
 4. Add entry to ``CONTROL_ROOM_IPS`` in ``gracedb/config/settings/base.py``
 5. Add signoff option for X1 in ``gracedb/templates/gracedb/event_detail.html``

docs/admin_docs/source/new_gracedb_instance.rst

@@ -170,12 +170,12 @@ On the new server, as yourself, import the database using the ``gracedb`` user's
 
 Note that files related to the events aren't part of the database and won't exist on the new server unless you copy them over, too (see :ref:`copying_event_data` for more information).
 
-Next, become the ``gracedb`` user, enter the Django manager shell, and delete all Contacts and Triggers so that people don't get phone or email alerts from this instance without signing up for them::
+Next, become the ``gracedb`` user, enter the Django manager shell, and delete all Contacts and Notifications so that people don't get phone or email alerts from this instance without signing up for them::
 
-    from userprofile.models import Contact, Trigger
-    for c in Contacts.objects.iterator():
+    from alerts.models import Contact, Notification
+    for c in Contact.objects.iterator():
         c.delete()
-    for t in Trigger.objects.iterator():
+    for t in Notification.objects.iterator():
         t.delete()
 
 You might want to delete the Events, too, especially if you copy the production database.

docs/admin_docs/source/phone_alerts.rst

@@ -81,7 +81,7 @@ Most of the relevant code is in ``gracedb/events/alerts.py``, including the foll
 - ``issueAlertForLabel``
 - ``issueEmailAlert``
 
-There is also some relevant code in ``gracedb/userprofile/models.py``, which defines a ``PhoneNumberField`` for the ``Contact`` model and validates the phone number when a user signs up for this service.
+There is also some relevant code in ``gracedb/alerts/fields.py``, which defines a ``PhoneNumberField`` for the ``Contact`` model and validates the phone number when a user signs up for this service.
 
 The TwiML bin SIDs are used in ``make_twilio_calls`` to generate the URLs and make the POST request.
 These SIDs, along with the Account SID and Auth Token should **NOT** be saved in the git repository.

docs/admin_docs/source/public_gracedb.rst

@@ -114,7 +114,7 @@ example, here is how to grant ``view`` permissions to ``public``::
     group_name = 'public'
     perm_shortname = 'view'
 
-    url = g.service_url + urllib.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
+    url = g.service_url + urllib.parse.quote('events/%s/%s/%s' % (graceid, group_name, perm_codename))
     r = g.put(url)
 
 Templates 

docs/admin_docs/source/robot_certificate.rst

@@ -103,13 +103,14 @@ Edit the migration to do what you want it to do. You could use this as a templat
     ]
 
     def create_robots(apps, schema_editor):
-        RobotUser = apps.get_model('ligoauth', 'RobotUser')
+        User = apps.get_model('auth', 'User')
         X509Cert = apps.get_model('ligoauth', 'X509Cert')
-        Group = apps.get_model('auth', 'Group')
-        lvc_group = Group.objects.get(name=settings.LVC_GROUP)
+        AuthGroup = apps.get_model('ligoauth', 'AuthGroup')
+        lvc_group = AuthGroup.objects.get(name=settings.LVC_GROUP)
+        robot_group = AuthGroup.objects.get(name='robot_accounts')
 
         for entry in ROBOTS:
-            user, created = RobotUser.objects.get_or_create(username=entry['username'])
+            user, created = User.objects.get_or_create(username=entry['username'])
             if created:
                 user.first_name = entry['first_name']
                 user.last_name = entry['last_name']
@@ -121,10 +122,8 @@ Edit the migration to do what you want it to do. You could use this as a templat
 
             # Create the cert objects and link them to our user.
             for dn in entry['dns']:
-                cert, created = X509Cert.objects.get_or_create(subject=dn)
-                if created:
-                    cert.save()
-                cert.users.add(user)
+                cert, created = X509Cert.objects.get_or_create(subject=dn,
+                    user=user)
 
             # Add our user to the LVC group. This permission is required to 
             # do most things, but may *NOT* always be appropriate. It may
@@ -132,14 +131,17 @@ Edit the migration to do what you want it to do. You could use this as a templat
             # a particular pipeline.
             lvc_group.user_set.add(user)
 
+            # Add user to robot accounts
+            robot_group.user_set.add(user)
+
     def delete_robots(apps, schema_editor):
-        RobotUser = apps.get_model('ligoauth', 'RobotUser')
+        User = apps.get_model('auth', 'User')
         X509Cert = apps.get_model('ligoauth', 'X509Cert')
 
         for entry in ROBOTS:
             for dn in entry['dns']:
                 X509Cert.objects.get(subject=dn).delete()
-            RobotUser.objects.get(username=entry['username']).delete()
+            User.objects.get(username=entry['username']).delete()
 
     class Migration(migrations.Migration):
 

docs/admin_docs/source/server_maintenance.rst

@@ -181,6 +181,6 @@ See :ref:`sql_tips` for how to do this.
 
 A few things that you may want to do after copying the database, but before beginning your debugging:
 
-* Turn off phone alerts! Obviously the Contact and Trigger instances are part of the database you just copied and will trigger and annoy people if you submit events for testing.  The easiest solution is probably to just delete all of the Contact and/or Trigger objects (in the copied database) through the Django shell.
+* Turn off phone alerts! Obviously the Contact and Notification instances are part of the database you just copied and will trigger and annoy people if you submit events for testing.  The easiest solution is probably to just delete all of the Contact and/or Notification objects (in the copied database) through the Django shell.
 * You may want to turn off XMPP alerts just to be safe.