Merge branch 'new-look'

.gitlab-ci.yml

@@ -17,7 +17,7 @@ before_script:
   - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
 .test: &test
-  image: igwn/base:stretch
+  image: igwn/base:buster
   services:
     - mysql:5.6
   variables:
@@ -58,6 +58,11 @@ before_script:
           libssl-dev
           libxml2-dev
           swig
+          pkg-config 
+          libpng-dev 
+          libfreetype6-dev 
+          libmariadb-dev-compat 
+          libxslt-dev
           ${PYTHON}-pip
     # install everything else from pip
     - ${PYTHON} -m pip install -r requirements.txt
@@ -83,7 +88,7 @@ before_script:
       - .cache/pip
       - .cache/apt
 
-test:3.5:
+test:3.7:
   <<: *test
 
 branch_image:

Dockerfile

-FROM igwn/base:stretch
+FROM igwn/base:buster
 LABEL name="LIGO GraceDB Django application" \
-      maintainer="tanner.prestegard@ligo.org" \
-      date="20190920"
+      maintainer="alexander.pace@ligo.org" \
+      date="20200807"
 ARG SETTINGS_MODULE="config.settings.container.dev"
 
 COPY docker/SWITCHaai-swdistrib.gpg /etc/apt/trusted.gpg.d
 COPY docker/backports.pref /etc/apt/preferences.d
-RUN echo 'deb http://pkg.switch.ch/switchaai/debian stretch main' > /etc/apt/sources.list.d/shibboleth.list
-RUN echo 'deb http://deb.debian.org/debian stretch-backports main' > /etc/apt/sources.list.d/backports.list
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+RUN echo 'deb http://pkg.switch.ch/switchaai/debian buster main' > /etc/apt/sources.list.d/shibboleth.list
+RUN echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/backports.list
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
 RUN apt-get update && \
     apt-get install --install-recommends --assume-yes \
         apache2 \
@@ -19,24 +19,30 @@ RUN apt-get update && \
         libldap2-dev \
         libsasl2-dev \
         libxml2-dev \
+        pkg-config \
+        libpng-dev \
+        libfreetype6-dev \
+        libmariadb-dev-compat \
+        libxslt-dev \
         libsqlite3-dev \
         ligo-ca-certs \
         mariadb-client \
         nodejs \
         osg-ca-certs \
-        python3.5 \
-        python3.5-dev \
+        python3.7 \
+        python3.7-dev \
         python3-libxml2 \
         python3-pip \
         procps \
         shibboleth \
-        supervisor \
         libssl-dev \
         swig \
         htop \
         telnet \
         vim && \
     apt-get clean && \
+    curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+    apt-get update && apt-get install --assume-yes yarn && \
     npm install -g bower
 
 COPY docker/entrypoint /usr/local/bin/entrypoint
@@ -66,8 +72,11 @@ RUN pip3 install --upgrade pip
 RUN pip3 install --upgrade setuptools wheel && \
     pip3 install -r requirements.txt
 
+# install supervisor from pip
+RUN pip3 install supervisor
+
 # Give pip-installed packages priority over distribution packages
-ENV PYTHONPATH /usr/local/lib/python3.5/dist-packages:$PYTHONPATH
+ENV PYTHONPATH /usr/local/lib/python3.7/dist-packages:$PYTHONPATH
 ENV ENABLE_SHIBD false
 ENV ENABLE_OVERSEER true
 ENV VIRTUAL_ENV dummy
@@ -116,4 +125,4 @@ RUN chmod 0755 /usr/local/bin/entrypoint && \
     find /app/gracedb_project -type f -exec chmod 0644 {} +
 
 ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

bower.json

@@ -8,7 +8,13 @@
     "jquery-ui": "1.12.1",
     "moment-timezone": "0.5.0",
     "moment": "2.11.1",
-    "tablesaw": "3.1.2"
+    "tablesaw": "3.1.2",
+    "materialize": "1.0.0",
+    "bootstrap": "4.1.0",
+    "datatables.net": "^1.10.20",
+    "datatables.net-bs4": "^3.2.2",
+    "ekko-lightbox": "^5.3.0",
+    "trumbowyg": "2.21.0"
   },
   "resolutions": {
     "webcomponentsjs": "^0.6.0"

config/settings/base.py

@@ -25,8 +25,12 @@ def get_from_env(envvar, default_value=None, fail_if_not_found=True):
 MAINTENANCE_MODE = False
 MAINTENANCE_MODE_MESSAGE = None
 
+# Enable/Disable Information Banner:
+INFO_BANNER_ENABLED = False
+INFO_BANNER_MESSAGE = "TEST MESSAGE"
+
 # Version ---------------------------------------------------------------------
-PROJECT_VERSION = '2.10.0'
+PROJECT_VERSION = '2.10.0-1'
 
 # Unauthenticated access ------------------------------------------------------
 # This variable should eventually control whether unauthenticated access is
@@ -39,7 +43,7 @@ UNAUTHENTICATED_ACCESS = True
 DEBUG = False
 
 # Number of results to show on latest page
-LATEST_RESULTS_NUMBER = 50
+LATEST_RESULTS_NUMBER = 25
 
 # Path to root URLconf
 ROOT_URLCONF = '{module}.urls'.format(module=os.path.basename(CONFIG_ROOT))

config/settings/container/base.py

@@ -73,6 +73,19 @@ if (isinstance(maintenance_mode, str) and
 MAINTENANCE_MODE_MESSAGE = \
     get_from_env('DJANGO_MAINTENANCE_MODE_MESSAGE', fail_if_not_found=False)
 
+# Get info banner settings from environment
+info_banner_enabled = get_from_env(
+    'DJANGO_INFO_BANNER_ENABLED',
+    default_value=False,
+    fail_if_not_found=False
+)
+# fix for other booleans:
+if (isinstance(info_banner_enabled, str) and
+    info_banner_enabled.lower() in ['true','t','1']):
+    INFO_BANNER_ENABLED = True
+INFO_BANNER_MESSAGE = \
+    get_from_env('DJANGO_INFO_BANNER_MESSAGE', fail_if_not_found=False)
+
 # Get email settings from environment
 EMAIL_BACKEND = 'django_ses.SESBackend'
 AWS_SES_ACCESS_KEY_ID = get_from_env('AWS_SES_ACCESS_KEY_ID')
@@ -84,6 +97,46 @@ AWS_SES_REGION_ENDPOINT = get_from_env('AWS_SES_REGION_ENDPOINT',
 AWS_SES_AUTO_THROTTLE = 0.25
 ALERT_EMAIL_FROM = get_from_env('DJANGO_ALERT_EMAIL_FROM')
 
+# AWS Elasticache settings:
+AWS_ELASTICACHE_ADDR = get_from_env('DJANGO_AWS_ELASTICACHE_ADDR')
+#CACHES['default'] = {
+#        'BACKEND': 'django_elasticache.memcached.ElastiCache',
+#        'LOCATION': AWS_ELASTICACHE_ADDR,
+#        'OPTIONS': {
+#            'IGNORE_CLUSTER_ERRORS': True,
+#        },
+#    }
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'LOCATION': AWS_ELASTICACHE_ADDR,
+    },
+    # For API throttles
+    'throttles': {
+        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
+        'LOCATION': 'api_throttle_cache', # Table name
+    },
+}
+
+MIDDLEWARE = [
+    'core.middleware.maintenance.MaintenanceModeMiddleware',
+    'events.middleware.PerformanceMiddleware',
+    'core.middleware.accept.AcceptMiddleware',
+    'core.middleware.api.ClientVersionMiddleware',
+    'core.middleware.api.CliExceptionMiddleware',
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
+    'core.middleware.proxy.XForwardedForMiddleware',
+    'user_sessions.middleware.SessionMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
+    'ligoauth.middleware.ControlRoomMiddleware',
+]
+
+
 
 
 # Priority server settings ----------------------------------------------------

config/settings/container/dev.py

@@ -74,7 +74,8 @@ if sentry_dsn is not None:
 # Home page stuff
 INSTANCE_TITLE = 'GraceDB Development Server'
 INSTANCE_INFO = """
-<h3>Development Instance</h3>
+<h5>Development Instance</h5>
+<hr>
 <p>
 This GraceDB instance is designed for GraceDB maintainers to develop and
 test in the AWS cloud architecture. There is <b>no guarantee</b> that the

config/settings/container/playground.py

@@ -25,7 +25,7 @@ ALLOWED_HOSTS += ['testserver']
 # Home page stuff
 INSTANCE_TITLE = 'GraceDB Playground'
 INSTANCE_INFO = """
-<h3>Playground instance</h3>
+<h5>Playground instance</h5>
 <p>
 This GraceDB instance is designed for users to develop and test their own
 applications. It mimics the production instance in all but the following ways:

config/settings/container/test.py

@@ -74,7 +74,8 @@ if sentry_dsn is not None:
 # Home page stuff
 INSTANCE_TITLE = 'GraceDB Testing Server'
 INSTANCE_INFO = """
-<h3>Testing Instance</h3>
+<h5>Testing Instance</h5>
+<hr>
 <p>
 This GraceDB instance is designed for Quality Assurance (QA) testing and
 validation for GraceDB and electromagnetic follow-up (EMFollow) developers.

config/settings/vm/base.py

@@ -61,23 +61,6 @@ MIDDLEWARE = [
     'ligoauth.middleware.ShibbolethWebAuthMiddleware',
     'ligoauth.middleware.ControlRoomMiddleware',
 ]
-#MIDDLEWARE = [
-#    'core.middleware.maintenance.MaintenanceModeMiddleware',
-#    'events.middleware.PerformanceMiddleware',
-#    'core.middleware.accept.AcceptMiddleware',
-#    'core.middleware.api.ClientVersionMiddleware',
-#    'core.middleware.api.CliExceptionMiddleware',
-#    'django.middleware.cache.UpdateCacheMiddleware',
-#    'django.middleware.common.CommonMiddleware',
-#    'django.middleware.cache.FetchFromCacheMiddleware',
-#    'django.contrib.sessions.middleware.SessionMiddleware',
-#    'core.middleware.proxy.XForwardedForMiddleware',
-#    'user_sessions.middleware.SessionMiddleware',
-#    'django.contrib.messages.middleware.MessageMiddleware',
-#    'django.contrib.auth.middleware.AuthenticationMiddleware',
-#    'ligoauth.middleware.ShibbolethWebAuthMiddleware',
-#    'ligoauth.middleware.ControlRoomMiddleware',
-#]
 
 # Set caches:
 CACHES = {

docker/shibboleth-ds/idpselect.css

 /* Top level is idpSelectIdPSelector */
 #idpSelectIdPSelector
 {
-    width: 512px;
-    text-align: left;
+    width:fit-content;
+    text-align: center;
     background-color: #FFFFFF;
     border: 2px #A40000 solid;
     padding: 10px;
+    margin-top:25px;
+    margin-bottom:25px;
 }
 
 /* Next down are the idpSelectPreferredIdPTile, idpSelectIdPEntryTile & idpSelectIdPListTile */
@@ -16,7 +18,7 @@
  */
 #idpSelectPreferredIdPTile
 {
-    height:138px; /* Force the height so that the  selector box
+    height: 138px /* Force the height so that the  selector box
                    * goes below when there is only one preslect 
                    */
 }

docs/user_docs/source/_templates/layout.html

-{% extends "!layout.html" %}
-
-{% block extrahead %}
-
-<link rel="stylesheet" href="_static/gracedb-nav-style.css" />
-<script src="/static/dojo/dojo.js" data-dojo-config="async: true"></script>
-<script>
-
-var getKeys = function(obj){
-   var keys = [];
-   for(var key in obj){
-      keys.push(key);
-   }
-   return keys;
-}
-
-require([
-    'dojo/_base/declare',
-    'dojo/query',
-    'dojo/parser',
-    'put-selector/put',
-    'dojo/dom',
-    'dojo/dom-construct',
-    'dojo/dom-style',
-    'dojo/request',
-    'dojo/NodeList-dom',
-    'dojo/NodeList-traverse',
-    'dojo/domReady!',
-], function(declare, query, parser, put, dom, domConstruct, domStyle, request) {
-
-    parser.parse();
-
-    // The url will look like: base + /documentation/...
-    var loc = window.location.href;
-    var ind = loc.indexOf('documentation');
-    var url = loc.substring(0,ind);
-    url += 'navbar_only';
-    
-    var header_div = dom.byId("gracedb-nav-header");
-
-    request.get(url).then(
-        function(text) {
-            var node = domConstruct.toDom(text);
-            var nl = query('*', node);
-            var header_content = "";
-            // XXX this should not be necessary. Why can't I just query directly for the node with
-            // id == 'content'?
-            nl.forEach(function(n) {
-                if (n.tagName == 'DIV' && n.id == 'content') {
-                    header_content = n.innerHTML;
-                }
-            });
-            header_div.innerHTML = header_content;
-        },
-        function(error) {
-            console.log("failed to get navbar content.")
-        }
-    );
-
-});
-
-</script>
-
-{% endblock %}
-
-{% block header %}
-
-<div id="gracedb-nav-header"></div>
-
-{% endblock %}

docs/user_docs/source/auth.rst

@@ -41,7 +41,8 @@ Creating new events in a *non-test* group (e.g., CBC or Burst) requires
 a special ``populate`` permission on the relevant pipeline object. These 
 permissions are set at the user (rather than group) level and are 
 maintained by hand. Send email to the GraceDB maintainer 
-or the ``uwm-help`` queue if you need a new Pipeline or pipeline permission.
+or the `IGWN Computing Helpdesk <mailto:computing-help@igwn.org>`__ 
+if you need a new pipeline or pipeline permission.
 
 Robot certificates
 =====================

docs/user_docs/source/conf.py

@@ -16,6 +16,9 @@ import sys
 import os
 import shlex
 
+# Import bootstrap theme:
+import sphinx_rtd_theme
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -30,12 +33,13 @@ import shlex
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    "sphinx_rtd_theme",
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+#templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
@@ -50,7 +54,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'GraceDB'
-copyright = u'2019, Tanner Prestegard, Alexander Pace, Brian Moe, Branson Stephens, Patrick Brady'
+copyright = u'2020, Tanner Prestegard, Alexander Pace, Brian Moe, Branson Stephens, Patrick Brady'
 author = u'Tanner Prestegard, Alexander Pace, Brian Moe, Branson Stephens, Patrick Brady'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -111,15 +115,20 @@ todo_include_todos = True
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'alabaster'
+sys.path.append(os.path.abspath('_themes'))
+html_theme_path = ['_themes']
+html_theme = 'sphinx_rtd_theme'
+#html_translator_class = 'bootstrap.HTMLTranslator'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+html_theme_options = {
+    'navbar_site_name': "Documentation Contents", 
+}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+html_theme_path = sphinx_rtd_theme.get_html_theme_path()
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".

docs/user_docs/source/general.rst

@@ -57,4 +57,5 @@ This documentation is not as great as it could be, but we are working on it.
 
 LIGO/Virgo users can join the GraceDB channel in the collaboration's Mattermost instance or email the DASWG mailing list for help.
 
-To report a problem, either `post an issue <https://git.ligo.org/lscsoft/gracedb/issues>`__ or send mail to uwm-help@ligo.org.
+To report a problem, either `post an issue <https://git.ligo.org/lscsoft/gracedb/issues>`__ or email the 
+`IGWN Computing Helpdesk <mailto:computing-help@igwn.org>`__.

docs/user_docs/source/index.rst

@@ -3,7 +3,8 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to GraceDB's documentation!
+===================================
+GraceDB Documentation
 ===================================
 
 Contents:
@@ -18,10 +19,3 @@ Contents:
    Report a bug (LIGO/Virgo users) <https://git.ligo.org/lscsoft/gracedb/issues>
 
 
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-

docs/user_docs/source/lvem.rst

@@ -170,7 +170,8 @@ observation records:
 
 If you discover a mistake in your observation record, the best way to correct
 it is to submit a new observation record with corrected values and request that
-the old one be deleted. Please send an email to uwm-help@cgca.uwm.edu with
+the old one be deleted. Please send an email to the
+`IGWN Computing Helpdesk <mailto:computing-help@igwn.org>`__ with
 something like "delete GraceDB EMObservation" in the subject line. Tell us
 which entry you'd like deleted, and we'll take care of it.  In the future, we
 are hoping to make these observation records editable by the submitter.

docs/user_docs/source/responding_to_lvalert.rst

@@ -32,10 +32,11 @@ Registering to receive LVAlert messages
 =======================================
 
 LSC-Virgo members can activate accounts by simply completing the form 
-`here <https://www.lsc-group.phys.uwm.edu/cgi-bin/jabber-acct.cgi>`__.
+`here <https://lvalert-test.cgca.uwm.edu>`__.
 
 If you need to create an account that is not attached to your user.name, you
-can email uwm-help@cgca.uwm.edu and request an account.  Once you have created an
+can email `IGWN Computing Helpdesk <mailto:computing-help@igwn.org>`__ 
+and request an account.  Once you have created an
 account, you will be able to subscribe the account to different pubsub nodes
 and receive lvalert messages.
 
@@ -404,8 +405,8 @@ these events is available `here <https://gracedb.ligo.org/events/search/?query=g
 like G184098 (the cWB entry for GW150914) or others with low FAR.*
 Instead, please use a test event as described above. There are also test instances of
 GraceDB available if you'd prefer to not work with the production server right
-away. Contact uwm-help@cgca.uwm.edu with a descriptive subject line for more
-information.
+away. Contact `IGWN Computing Helpdesk <mailto:computing-help@igwn.org>`__  
+with a descriptive subject line for more information.
 
 At this point, you're pretty much ready to go. However, I'll leave you with one
 more example for what ``iReact.py`` might look like::

gracedb/api/v1/events/views.py

@@ -867,7 +867,6 @@ class EventLogList(InheritPermissionsAPIView):
 
     @event_and_auth_required
     def post(self, request, event):
-        #raise ValueError(request.data, event)
         message = request.data.get('comment')
         # Handle requests encoded as multipart/form or regular JSONs
         if isinstance(request.data, QueryDict):

gracedb/events/templatetags/mediaviews.py

+from django import template
+from django.conf import settings
+
+from django.utils.html import conditional_escape
+from django.utils.safestring import mark_safe
+
+#Django filtering stuff:
+import operator
+from django.db.models import Q
+from functools import reduce
+
+from events.models import Event, Tag
+from gracedb.core.urls import build_absolute_uri
+import os
+
+register = template.Library()
+
+# Define some stuff:
+blessed_tag_priority_order = [
+    'analyst_comments',
+    'psd',
+    'data_quality',
+    'sky_loc',
+    'background',
+    'ext_coinc',
+    'strain',
+    'tfplots',
+    'sig_info',
+    'audio',
+    'em_follow',
+]
+
+
+# This section is for image formatting purposes. I want to assign 
+# "blessed" file extensions, and then set up a django Q-filter. 
+# Then, define which images get displayed as high aspect-ratio. 
+# This is a little hacky, but I didn't take the time to learn how to 
+# incorporate javascript or the django.core.files.images models. If 
+# pipelines want to upload new images that are a different aspect ratio, 
+# add its name to the blessed list, or figure out a better way :-/
+
+img_file_extensions = ['.png','.jpg','.jpeg','.gif']
+images_filter = reduce(operator.or_, (Q(filename__contains=ex) for ex in img_file_extensions))
+
+wide_image_names = ['omegascan','coherence', 'em_bright']
+wide_filter = reduce(operator.or_, (Q(filename__contains=name) for name in wide_image_names))
+
+#styles, etc
+
+button_template = """<button class="btn btn-primary" 
+                      type="button" 
+                      data-toggle="collapse" 
+                      data-target="#{}" 
+                      aria-expanded="true" 
+                      aria-controls="{}">
+                         {} 
+                      </button>
+"""
+
+embedded_button_template = """<button class="btn btn-secondary btn-sm" 
+                      type="button" 
+                      data-toggle="collapse" 
+                      data-target="#{}" 
+                      aria-expanded="true" 
+                      aria-controls="{}">
+                         Collapse {} 
+                      </button>
+"""
+collapsed_card_template = """
+<p><div class="collapse show" id="{}">
+  <div class="card card-body">
+    <div class="card-header log-card-header text-left">
+      <h6>{}</h6>
+    </div>
+   {}
+  </div>
+  <br>
+   {}
+</div></p>
+"""
+
+img_style_template = """max-height= 250px;
+"""
+
+image_card_div = """
+<div class="card my-3 log-comment-card" style="min-width:250px;">
+  <div class="card-header log-comment-card-header text-left">
+    <h7>Log Image</h7>
+  </div>
+  <a href="{}" data-toggle="lightbox" data-type="image" data-gallery="{}">
+  <img class="card-img-top img-fluid" src="{}" style="width:auto;"/>
+  </a>
+      <div class="card-body">
+        <hr width="50%"/>
+        {}
+      </div>
+    </div>
+"""
+
+comment_card_div = """
+<div class="card my-3 log-comment-card">
+  <div class="card-header log-comment-card-header text-left">
+    <h7>Log Comment</h7>
+  </div>
+  <div class="card-body">
+    <p class="card-text">{}</p>
+    <hr width="50%"/>
+    <footer class="blockquote-footer">{}</footer>
+  </div>
+</div>
+"""
+
+image_card_caption = """{} <footer class="blockquote-footer"> Submitted by {} on {} </footer>"""
+
+comment_card_title = """Submitted by {} on {} """
+
+def card_content(tagged_log_list, tag_name):
+    # Takes in a list of log messages that are tagged. 
+    # First deal with images. Check if extension is in allowed
+    # list of extensions:
+    rv =""""""
+
+   # First, wide images:
+    for l in tagged_log_list.filter(images_filter & wide_filter):
+        rv += img_div(l,tag_name)