diff --git a/.gitignore b/.gitignore index c1b5e24029c97d63d9c5ee2521fbb848dd5c8800..c27b0959e99a10741f9b4e74f3e671dc45926291 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ django-*.wsgi static-collected static/admin/ static/rest_framework/ +doc/build/* +doc/build/.buildinfo diff --git a/doc/CHANGES b/doc/CHANGES deleted file mode 100644 index 1f3eabb4df3d6a73ee013a8fba1d70d773b5ab9f..0000000000000000000000000000000000000000 --- a/doc/CHANGES +++ /dev/null @@ -1,71 +0,0 @@ -================================================================== -ER2 - - settings refactor - More unified settings/config. - settings/__init__.py - locates and imports * from settings.default then settings.WHATEVER - settings/defaut.py - /production.py - /development.py - /misc.py - - Simple enough and gets rid of settings_dev crap. - Keeps prod/dev configs from diverging too much. - -================================================================== -Branch goocharts - -================================================================== -Branch Master -Django 1.1 -> 1.3 -Tue Dec 20 14:32:32 CST 2011 - -Deprecation warnings in apache log. /var/log/apache2/error.log - - [Tue Dec 20 10:28:11 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/db/__init__.py:19: DeprecationWarning: settings.DATABASE_* is deprecated; use settings.DATABASES instead. - [Tue Dec 20 10:28:11 2011] [error] DeprecationWarning - [Tue Dec 20 10:28:11 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/db/__init__.py:60: DeprecationWarning: Short names for ENGINE in database configurations are deprecated. Prepend default.ENGINE with 'django.db.backends.' - [Tue Dec 20 10:28:11 2011] [error] DeprecationWarning - -Changed settings.py to new syntax. - ------------------ - - [Tue Dec 20 10:28:12 2011] [error] /usr/lib/python2.6/dist-packages/pytz/__init__.py:32: UserWarning: Module _mysql was already imported from /usr/lib/pymodules/python2.6/_mysql.so, but /usr/lib/pymodules/python2.6 is being added to sys.path - [Tue Dec 20 10:28:12 2011] [error] from pkg_resources import resource_stream - -Ugh. This is a bug in distribute/pkg_resources, apparently. Did a workaround -- an early import of -pkg_resources will prevent this, so there is a pointless import at the beginning of settings.py. -This seemed to rid us of this message. - ------------------ - - [Tue Dec 20 10:28:12 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/contrib/auth/__init__.py:26: DeprecationWarning: Authentication backends without a `supports_object_permissions` attribute are deprecated. Please define it in gracedb.middleware.auth.LigoAuthBackend. - [Tue Dec 20 10:28:12 2011] [error] DeprecationWarning) - [Tue Dec 20 10:28:12 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/contrib/auth/__init__.py:31: DeprecationWarning: Authentication backends without a `supports_anonymous_user` attribute are deprecated. Please define it in gracedb.middleware.auth.LigoAuthBackend. - [Tue Dec 20 10:28:12 2011] [error] DeprecationWarning) - -Modified gracdeb.middleware.auth.LigoAuthBackend to have supports_object_permissions -and supports_anonymous_user class attributes. - ------------------ - - [Tue Dec 20 10:28:12 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/core/context_processors.py:27: DeprecationWarning: The context processor at `django.core.context_processors.auth` is deprecated; use the path `django.contrib.auth.context_processors.auth` instead. - [Tue Dec 20 10:28:12 2011] [error] DeprecationWarning - -Replaced in settings.py / settings_dev.py - ------------------ - - [Tue Dec 20 10:28:12 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/template/loaders/filesystem.py:58: DeprecationWarning: 'django.template.loaders.filesystem.load_template_source' is deprecated; use 'django.template.loaders.filesystem.Loader' instead. - [Tue Dec 20 10:28:12 2011] [error] DeprecationWarning - -Replaced in settings.py / settings_dev.py - ------------------ - -error.log:[Thu Dec 22 14:40:20 2011] [error] /home/lars/wsgi-sandbox/lib/python2.6/site-packages/django/contrib/syndication/views.py:194: DeprecationWarning: The syndication feed() view is deprecated. Please use the new class based view API. - -Done. Fixed in gracedb/feeds.py and urls.py. -================================================================== diff --git a/doc/README b/doc/README deleted file mode 100644 index 259cd617190f3c43a3a7eee6987da3e088eaf43e..0000000000000000000000000000000000000000 --- a/doc/README +++ /dev/null @@ -1,61 +0,0 @@ - -Report generation. - ./manage.py make_histograms -================================================================== - -Requirements: - gnuplot >= v4.2 - glue (python-glue in debian) - python-pyxmpp - - mysql-server - MySQLdb-python (python-mysqldb in debian) - mod_wsgi (libapache2-mod-wsgi) - mod_auth_kerb (libapache2-mod-auth-kerb) - - voevent - -================================================================== - - -# ligo-shibboleth-metadata-3.2.0-1 - ligo-shibboleth-sp-2.0.0-1 - shibboleth-2.4.3-2.2 -# liblog4shib1-1.0.4-2.1 - - Apache must (should?) use MPM prefork and NOT worker model. - (debian - apache2-mpm-prefork) - This is because the Django ORM is (probably? possibly?) not threadsafe, - or maybe it probably *is* threadsafe... hard to tell, but prefork - would be safer. - - http://groups.google.com/group/django-developers/browse_thread/thread/905f79e350525c95 - http://code.djangoproject.com/ticket/1442 - -We need glue.ligolw, which requires SOOO many things from pylal - it's just easier to install pylal all of LAL. - - - python-virtualenv - - - pip install django - pip install pytz - pip install simplejson - pip install django-maintenancemode - -nfs0001 - certs - apache config - install services - TAKE NOTES FOR CFENGINE -- ANYTHING outside of /home - -clients/configs ... remove references to archie / push to repos. - - -CREATE DATABASE gracedb; -CREATE DATABASE gracedb_test; -GRANT ALL PRIVILEGES ON gracedb.* TO 'gracedb' IDENTIFIED BY 'redrum4x'; -GRANT ALL PRIVILEGES ON gracedb_test.* TO 'gracedb' IDENTIFIED BY 'redrum4x'; -FLUSH PRIVILEGES; - diff --git a/doc/schema-1.0-1.1.sql b/doc/schema-1.0-1.1.sql deleted file mode 100644 index 70e385f6ca65322e920bf37ea3e9736e63ce7886..0000000000000000000000000000000000000000 --- a/doc/schema-1.0-1.1.sql +++ /dev/null @@ -1,4 +0,0 @@ - -ALTER TABLE gracedb_event -ADD COLUMN far double AFTER gpstime -; diff --git a/doc/schema-1.1-1.2.sql b/doc/schema-1.1-1.2.sql deleted file mode 100644 index dc1615f49e4a52e8a802d0b186af4fa5ff38438e..0000000000000000000000000000000000000000 --- a/doc/schema-1.1-1.2.sql +++ /dev/null @@ -1,4 +0,0 @@ - -ALTER TABLE gracedb_eventlog -MODIFY COLUMN comment LONGTEXT NOT NULL -; diff --git a/doc/schema-1.2-1.3.sql b/doc/schema-1.2-1.3.sql deleted file mode 100644 index a9f5d176dde9e9d4b32542e3991ccf9b11d7844d..0000000000000000000000000000000000000000 --- a/doc/schema-1.2-1.3.sql +++ /dev/null @@ -1,5 +0,0 @@ - -ALTER TABLE userprofile_trigger -ADD COLUMN farThresh double AFTER triggerType -; - diff --git a/doc/source/_static/gracedb-nav-style.css b/doc/source/_static/gracedb-nav-style.css new file mode 100644 index 0000000000000000000000000000000000000000..8a612ba6a89f82e63a27042e232d3ae0748ca12b --- /dev/null +++ b/doc/source/_static/gracedb-nav-style.css @@ -0,0 +1,144 @@ + + +html, body { + color: black; + background-color: white; + margin: 0; + padding: 0; +} + +a.link, a, a.active { + color: #369; +} + +h1,h2,h3,h4,h5,h6,#getting_started_steps { + font-family: "Century Schoolbook L", Georgia, serif; + font-weight: bold; +} + +h1.docnav { + font-size: 25px; +} + +#getting_started_steps li { + font-size: 80%; + margin-bottom: 0.5em; +} + +#gracedb-nav-header { + color: black; + font-size: 127%; + background-color: white; + font: x-small "Lucida Grande", "Lucida Sans Unicode", geneva, verdana, sans-serif; +/* width: 757px; */ + width: 95%; + margin: 0 auto 0 auto; + border: none; +/* border-left: 1px solid #aaa; */ +/* border-right: 1px solid #aaa; */ + padding: 10px 10px 0px 10px; +} + + +/* Nav */ +#nav { + margin:0; + padding:0; + background:#eee; /* Nav base color */ + float: left; + width: 100%; + font-size: 13px; + border:1px solid #42432d; + border-width:1px 1px; +} + +#nav #nav-user +{ + color:#000; + background:#eee; /* Nav base color */ + padding:4px 20px 4px 20px; + float: right; + width: auto; + text-decoration:none; + font:bold 1em/1em Arial, Helvetica, sans-serif; + text-transform:uppercase; + /* text-shadow: 2px 2px 2px #555; */ +} + +#nav #nav-login +{ + float: right; +} + +#nav li { + display:inline; + padding:0; + margin:0; +} +/* +#nav li:first-child a { + border-left:1px solid #42432d; +} +*/ + +#nav a:link, +#nav a:visited { + color:#000; + background:#eee; /* Nav base color */ + /* padding:20px 40px 4px 10px; */ + padding:4px 20px 4px 20px; + float: left; + width: auto; + border-right:1px solid #42432d; + + text-decoration:none; + font:bold 1em/1em Arial, Helvetica, sans-serif; + text-transform:uppercase; + /* text-shadow: 2px 2px 2px #555; */ +} +#nav a:hover { + /* color:#fff; / * Use if bg is dark */ + background: #dce2ed; /* Nav hover color */ +} + +#home #nav-home a, +#create #nav-create a, +#search #nav-search a, +#doc #nav-doc a, +#reports #nav-reports a, +#feeds #nav-feeds a, +#about #nav-about a, +#archive #nav-archive a, +#lab #nav-lab a, +#reviews #nav-reviews a, +#userprofile #nav-userprofile a, +#latest #nav-latest a, +#contact #nav-contact a { + background: #a9b0ba; /* Nav selected color */ + /* color:#fff; / * Use if bg is dark */ + /* text-shadow:none; */ +} +#home #nav-home a:hover, +#create #nav-create a, +#search #nav-search a, +#doc #nav-doc a, +#reports #nav-reports a, +#feeds #nav-feeds a, +#about #nav-about a:hover, +#archive #nav-archive a:hover, +#lab #nav-lab a:hover, +#reviews #nav-reviews a:hover, +#userprofile #nav-userprofile a:hover, +#latest #nav-latest a:hover, +#contact #nav-contact a:hover { + /* background:#e35a00; */ + background: #a9b0ba; /* Nav selected color */ +} +#nav a:active { + /* background:#e35a00; */ + background: #a9b0ba; /* Nav selected color */ + color:#fff; +} + + + diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html new file mode 100644 index 0000000000000000000000000000000000000000..e7510e13af926a63d42f8fcc6fb0331eb87c86de --- /dev/null +++ b/doc/source/_templates/layout.html @@ -0,0 +1,69 @@ +{% extends "!layout.html" %} + +{% block extrahead %} + +<link rel="stylesheet" href="_static/gracedb-nav-style.css" /> +<script src="/bower-static/dojo/dojo.js" data-dojo-config="async: true"></script> +<script> + +var getKeys = function(obj){ + var keys = []; + for(var key in obj){ + keys.push(key); + } + return keys; +} + +require([ + 'dojo/_base/declare', + 'dojo/query', + 'dojo/parser', + 'dojo/dom', + 'dojo/dom-construct', + 'dojo/dom-style', + 'dojo/request', + 'dojo/NodeList-dom', + 'dojo/NodeList-traverse', + 'dojo/domReady!', +], function(declare, query, parser, dom, domConstruct, domStyle, request) { + + parser.parse(); + + // The url will look like: base + /documentation/... + var loc = window.location.href; + var ind = loc.indexOf('documentation'); + var url = loc.substring(0,ind); + url += 'navbar_only'; + + var header_div = dom.byId("gracedb-nav-header"); + + request.get(url).then( + function(text) { + var node = domConstruct.toDom(text); + var nl = query('*', node); + var header_content = ""; + // XXX this should not be necessary. Why can't I just query directly for the node with + // id == 'content'? + nl.forEach(function(n) { + if (n.tagName == 'DIV' && n.id == 'content') { + header_content = n.innerHTML; + } + }); + header_div.innerHTML = header_content; + }, + function(error) { + console.log("failed to get navbar content.") + } + ); + +}); + +</script> + +{% endblock %} + +{% block header %} + +<div id="gracedb-nav-header"></div> + +{% endblock %} diff --git a/doc/source/auth.rst b/doc/source/auth.rst new file mode 100644 index 0000000000000000000000000000000000000000..e781f22555096009c712d4c01f97248c75032e2f --- /dev/null +++ b/doc/source/auth.rst @@ -0,0 +1,23 @@ +.. _auth: + +================================ +Authentication and Authorization +================================ + +GraceDB permissions +========================================= + +Under construction. + +Robot certificates +===================== + +.. Observe that robot certificates are not necessary for LV-EM MOU partners. Please see +.. section *link* concerning scripted access to GraceDB + +Under construction. + +Robot keytabs +====================== + +Under construction. diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..170cd93c35e4c25c80ba47c98c0771dc52270dd2 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# +# GraceDB documentation build configuration file, created by +# sphinx-quickstart on Mon Jun 15 09:22:37 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.todo', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'GraceDB' +copyright = u'2015, Brian Moe, Branson Stephens, Patrick Brady' +author = u'Brian Moe, Branson Stephens, Patrick Brady' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.19.dev0' +# The full version, including alpha/beta/rc tags. +release = '1.19.dev0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GraceDBdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'GraceDB.tex', u'GraceDB Documentation', + u'Brian Moe, Branson Stephens, Patrick Brady', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'gracedb', u'GraceDB Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'GraceDB', u'GraceDB Documentation', + author, 'GraceDB', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/doc/source/general.rst b/doc/source/general.rst new file mode 100644 index 0000000000000000000000000000000000000000..4a4d1332e6beed2f35ca73bbf9afbf1218742c0f --- /dev/null +++ b/doc/source/general.rst @@ -0,0 +1,68 @@ +================ +Introduction +================ + +GraceDB in context +========================== + +GraceDB serves as a hub for communication among LVC data analysts and +electromagnetic astronomers, as well as a database for storing and displaying +information about candidate events: + +.. image:: images/annotations_flow_white.pdf + +The primary responsibilities of GraceDB are to: + +- ingest and store information about events +- expose that information to users and followup robots +- send alerts about each new piece of information + +As such, GraceDB curates content from many sources, but does not itself produce +any original scientific content. GraceDB is mainly geared toward low latency +data analysis +and electromagnetic followup, but need not be used exclusively for those +purposes. The diagram above depicts a typical sequence of events: + +#. An LVC data analysis pipeline detects and interesting candidate + gravitational wave (GW) event and submits it to GraceDB. +#. GraceDB sends an LVAlert message which notifies LVC followup + robots of the new event. +#. The followup robots perform analyses and report results back to + GraceDB. These results accumulate on the candidate event's page. +#. Eventually, the robot responsible for approving events for + EM followup forwards the event to `GCN/TAN <http://gcn.gsfc.nasa.gov>`_. +#. Astronomers receive a GCN Notice for the new event, perform followup + observations, and report results (as appropriate) via GraceDB. + +Overview of components +====================== + +GraceDB consists of the server (`gracedb.ligo.org <https://gracedb.ligo.org>`_) +and a set of client tools. Two user interfaces are available: the web interface +for browser access (e.g., the one you are using now), and the +`REST <https://en.wikipedia.org/wiki/Representational_state_transfer>`_ API. +These interfaces represent the information in GraceDB in different ways: +the web interface naturally represents information as HTML pages, whereas +the REST interface delivers JSON representations. + +The client tools (available via ``pip``, SL6 or Debian packages, and source +build, see :ref:`installing_the_client`) provide a way to interact via the REST API. +These tools include a client +class in Python which contains methods for most of the common GraceDB operations +There is also a ``gracedb`` tool for the command line that wraps +the Python client. + +The GraceDB API conforms to the RESTful principles of "uniform interface" and +"resource-oriented architecture," and so users experienced with such +interfaces may wish to create their own clients. This documentation will +briefly touch on building such clients (see :ref:`coding_against_api`). + +Where can I go for help? +================================== + +This documentation is not as great as it could be, but +we are working on it. For help with issues not addressed here, please +send mail to uwm-help@ligo.org. + +.. directions to docs for related LVAlert service + diff --git a/doc/source/images/annotations_flow_white.pdf b/doc/source/images/annotations_flow_white.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b93b696a0dfd199cfd95c5504fc22dad1694faf9 Binary files /dev/null and b/doc/source/images/annotations_flow_white.pdf differ diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..2559af4f323753a10b8ddfbab697ec3b3b1d90bd --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,29 @@ +.. GraceDB documentation master file, created by + sphinx-quickstart on Mon Jun 15 09:22:37 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GraceDB's documentation! +=================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + general + models + web + rest + lvalert + lvem + auth + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/lvalert.rst b/doc/source/lvalert.rst new file mode 100644 index 0000000000000000000000000000000000000000..4ceb5270db67471e9164896e47b72090c4d76fa9 --- /dev/null +++ b/doc/source/lvalert.rst @@ -0,0 +1,72 @@ +===================================== +Integration with LVAlert +===================================== + +Introduction +=============================================== + +GraceDB uses `LVAlert +<https://www.lsc-group.phys.uwm.edu/daswg/projects/lvalert.html>`_ to send +alerts to listeners within the LVC. The content of the LVAlert message is +designed to convey actionable information about a state change in GraceDB, +whether it involves the creation of a new event, or the updating or labeling of +an existing one. + +.. NOTE:: + An LVAlert message is sent out for *any* new event or annotation that arrives in the + GraceDB database. This means that + a message volumes may be very high under certain circumstances, and + appropriate filtering is required in order for LVAlert to be useful. + +Listening to specific event streams +============================================== + +When a user runs ``lvalert_listen``, she/he will receive messages over all **nodes** to +which she/he is subscribed. The node names consist of at least two elements:: + + <group_name>_<pipeline_name> + +In other words, the (lower-cased) names of the Group and Pipeline separated by an +underscore. For example, the node ``burst_cwb`` would catch all messages relating to +events in the Burst group from the cWB pipeline. One can also specify the search name:: + + <group_name>_<pipeline_name>_<search_name> + +which has the effect of narrowing down the messages to only those related to a specific +search. For example, the node ``burst_cwb_allsky`` will contain messages relating to the +AllSky search, but not the MDC search. GraceDB tries to send a message to all applicable +nodes. Thus, a message sent to the node ``burst_cwb_allsky`` will *also* be sent to the +node ``burst_cwb``. This property allows the user to filter by search at the level of +specifying LVAlert processing scripts for individual nodes. + +.. tell them how to get the names of all of the nodes + +LVAlert message contents +================================================ + +GraceDB sends messages as a JSON-encoded dictionary. The dictionary contains the +following keys: + +- ``uid``: the unique ID (a.k.a. ``graceid``) of the relevant event +- ``alert_type``: ``new``, ``update``, or ``label`` +- ``description``: a text description (if applicable) +- ``file``: a URL for the relevant file (if applicable) +- ``object``: a dictionary representing the relevant object + +For example, when a new event is created, an LVAlert message is created +with alert type ``new``, and the ``object`` is just the JSON representation of +of the event provided by the REST interface (see :ref:`searching_for_events`). + +.. examples of the different types +.. demonstration of how to parse + +.. what kind of alert does a replacement trigger? + +Further reading on LVAlert +===================================================== + +.. links to LVAlert documentation + +Further information on using LVAlert can be found on the +`LVAlert Project Page <https://www.lsc-group.phys.uwm.edu/daswg/projects/lvalert.html>`_ +and the `LVAlert Howto <https://www.lsc-group.phys.uwm.edu/daswg/docs/howto/lvalert-howto.html>`_. diff --git a/doc/source/lvem.rst b/doc/source/lvem.rst new file mode 100644 index 0000000000000000000000000000000000000000..93e622fb43d418410f9459f838549c833644735a --- /dev/null +++ b/doc/source/lvem.rst @@ -0,0 +1,79 @@ +============================= +Features for EM Collaboration +============================= + +On logging in +============= + +Users must be logged in for access to GraceDB events and for permission to +upload followup information. Please use the same identity you used when you +registered at gw-astronomy.org. + +.. _basic_auth_for_lvem: + +Scripted access for LV-EM members +============================================ + +.. Rationale: Non-LVC collaborators may not have ready access to robot certificates or keytabs. + +Under construction. + +Downloading a skymap +====================== + +Under construction. + +.. _create_emobservation: + +Reporting coordinates of followup observations +=============================================== + +In the following example, the GraceDB Python client is used to create an +observation record consisting of three separate footprints:: + + # Define the parameters of the observation to be reported + grace_id = 'M158044' # the event's UID + group = 'CRTS' # the MOU group + comment = 'hello my friend' # free text comment + raList = [123.0,124.0,125.0] # RAs of centers (degrees) + decList = [10.0,11.0,13.0] # Dec of centers (degrees) + startTimeList = [ # beginnings of exposures (UTC) + '2015-05-31T12:45:00', + '2015-05-31T12:49:00', + '2015-05-31T12:53:00'] + raWidthList = 10.0 # list (or one for all) of widths in RA (degrees) + decWidthList = 10.0 # list (or one for all) of widths in Dec (degrees) + durationList = 20.0 # list (or one for all) of exposure times in sec + + # Instantiate the GraceDB client + client = GraceDbBasic() + + # Write the EMObservation record to GraceDB + r = client.writeEMObservation(grace_id, group, raList, raWidthList, + decList, decWidthList, startTimeList, durationList, comment) + + if r.status == 201: # 201 means 'Created' + print 'Success!' + +To use the ``GraceDbBasic`` client, the user needs to already have a basic auth +password for scripted access, and to have put this in a protected ``.netrc`` file +(see :ref:`basic_auth_for_lvem`). + +For users not familiar with Python, there are several other options available for +uploading observation records: + +- by using the webform on each event page (scroll down to the 'EM Observations' + section and click on 'add observation record'). However, this method requires + by-hand data entry. + +- by ``curl``-ing directly against the EM observation + resource in the API (for an example, see + `here <https://gw-astronomy.org/wiki/LV_EM/CurlUploadFootprints>`_) + +- by coding against the GraceDB REST API + in one's own favorite language. If you choose to go this route, please + consider sending us your script or posting it in the LV-EM wiki Technical Info + page for the benefit of other users. See :ref:`coding_against_api`. + +- by email (not yet availabe, but in the works) + diff --git a/doc/source/models.rst b/doc/source/models.rst new file mode 100644 index 0000000000000000000000000000000000000000..ecbafa1da8a53f1f8d61a1a2ef5a9a1391b8e8df --- /dev/null +++ b/doc/source/models.rst @@ -0,0 +1,51 @@ +========================== +Data models +========================== + +What characterizes an event? +===================================== + +The different types of events in GraceDB are distinguished by the following parameters: + +- ``Group`` + the working group responsible for finding the candidate +- ``Pipeline`` + the data analysis software tool used make the detection +- ``Search`` + the search activity which led to the detection (often linked to a specific + astrophysical target, such as low-mass binaries) + +Specifying the values of these three parameters has the effect of isolating +an "event stream," e.g., *low-mass inspiral events detected by the gstlal +pipeline in the CBC group*. These categories were chosen in order avoid +situations in which events from different sources would overlap in searches +and alerts. + +The existing values for ``Group``, ``Pipeline``, and ``Search`` are as follows: + +.. Acknowledge slippage in the categories + +Base event model +==================================== + +.. In addition to the three parameters described above, there are additional +.. common attributes for all events. These are + +Under construction. + + +Event subclasses +==================================== + +Under construction. + +.. _annotation_models: + +Annotations +======================= + +.. explain what is an annotation + +.. explain about tags and blessed tags... need subsection? + +Under construction. diff --git a/doc/source/rest.rst b/doc/source/rest.rst new file mode 100644 index 0000000000000000000000000000000000000000..a9887a18704ef5ec8b75bec655c4a7be17e34b84 --- /dev/null +++ b/doc/source/rest.rst @@ -0,0 +1,269 @@ +========================== +Using the REST interface +========================== + +.. _installing_the_client: + +Installing the client +==================================== + +The GraceDB client tools should be installed already at the LVC computing clusters. +However, if you want to interact with GraceDB on your own machine, you will need +to install the client tools yourself. The easiest way is to use ``pip`` to install +it from the `Python Package Index <https://pypi.python.org/pypi>`_:: + + pip install ligo-gracedb + +(See `here <https://pip.pypa.io/en/latest/installing.html>`_ for instructions in +installing ``pip`` if it is not already available on your machine.) Additionally, +packages for Debian (``.deb``) and Scientific Linux (``.rpm``) are available by +pointing your system to the appropriate repositories as described +`here <https://www.lsc-group.phys.uwm.edu/daswg/download/repositories.html>`_. +Then the client tools can be installed via:: + + apt-get install python-ligo-gracedb + +or :: + + yum install ligo-gracedb + +Alternatively, the client can be built from source. Please note, however, that +the code at the head of the git repository may be volatile or unstable. The +repository may be cloned using LIGO credentials as follows:: + + git clone albert.einstein@ligo-vcs.phys.uwm.edu:/usr/local/git/gracedb-client.git + +Basic usage of the REST client +==================================== + +The documentation in this section will focus on the use of the Python REST client. The +alternative command-line client, ``gracedb`` is a simple wrapper on the REST client +provided for convenience (see below, :ref:`command_line_client`) and is not as +fully featured. + +.. NOTE:: + Before using the REST client, credentials for authentication must be available. + Run ligo-proxy-init or, if using a robot certificate, set the appropriate envronment + variables (for more infomation, see :ref:`auth`). + +The REST client is typically used in the Python interpreter or in a script to +accomplish a specific task, such as retrieving information about events matching +a query or creating a new event. The workflow involves importing the client class, +instantiating the client, and then calling the desired method:: + + from ligo.gracedb.rest import GraceDb, HTTPError + + client = GraceDb() + + try: + r = client.ping() + except HTTPError, e: + print e.message + + print "Response code: %d" % r.status + print "Response content: %s" % r.json() + + +In the above example, we merely ping the GraceDB server and examine the response. +If there is an error (such as an authentication failure), the ``HTTPError`` +exception will be thrown. If not, the response object contains a status code and +a response body in JSON. The ``json`` method on the response object simply decodes the +JSON content. In this particular case, the response code should be 200 (meaning "OK") +and the body contains a large dictionary +of information representing the `API Root resource <https://gracedb.ligo.org/apiweb/>`_. +Most of the examples below will ignore the error handling shown here for the +sake of brevity. + +In addition to ``ping``, the most important client methods are: + +- ``events`` for accessing a list of events, +- ``files`` for downloading a file or list of files, and ``writeFile`` for uploading, +- ``logs`` for obtaining a list of log entries, and ``writeLog`` to create a new one, +- ``emobservations`` for obtaining a list of EM followup observations, and + ``writeEMObservation`` to create a new one, +- ``labels``, ``writeLabel``, and ``removeLabel`` for managing labels, +- ``tags``, ``createTag``, and ``deleteTag`` for managing tags. + +Docstrings are available for most of the client methods. To see them, type +``help(client.ping)`` (for example) in the Python interpreter. + +.. I think we need some sort of note about how to use the GraceDbBasic client. + +.. _searching_for_events: + +Searching for events +=================================== + +Suppose you are working on a script to search for all events matching a +specific query and retrieve a piece of information about each event in +the search results. For example, the following code retrieves the chirp +mass for each gstlal event during ER5 with FAR less than 1.0E-4:: + + from ligo.gracedb.rest import GraceDb + client = GraceDb() + + # Retrieve an iterator for events matching a query. + events = client.events('gstlal ER5 far < 1.0e-4') + + # For each event in the search results, add the graceid + # and chirp mass to a dictionary. + results = {} + for event in events: + graceid = event['graceid'] + mchirp = event['extra_attributes']['CoincInspiral']['mchirp'] + results.update({ graceid: mchirp}) + +Note that the ``events`` method on the client returns an iterator on +event dictionaries rather than a list. The chirp mass is an attribute +specific the inspiral event subclass, hence the difference between accessing +the graceid and the chirp mass. + +But how did I know the structure of the event dictionary so that I could +pull out the chirp mass? The best way is to look at the structure of an +example event in the *browseable* REST API. Here some example events from +the different subclasses to demonstrate the structure of the event +dictionaries. + +- `Test gstlal MDC <https://gracedb.ligo.org/apiweb/events/T125738>`_ (a CBC event) +- `Test cWB MDC <https://gracedb.ligo.org/apiweb/events/T153811>`_ (a Burst event) +- `External Swift GRB <https://gracedb.ligo.org/apiweb/events/E160846>`_ (a GRB event) + +Creating new events +==================================== + +To create a new event, use the ``createEvent`` method on the client. In this +example, a new ``gstlal`` event is created in the ``Test`` group from a +file on disk:: + + from ligo.gracedb.rest import GraceDb + client = GraceDb() + + event_file = "/path/to/coinc.xml" + r = client.createEvent("Test","gstlal", event_file, search="LowMass") + event_dict = r.json() + graceid = event_dict["graceid"] + +The server response includes a JSON representation of the event, and the +event dictionary can thus be obtained as shown. In this example, the event +dictionary is used to retrieve the ``graceid`` of the new event. + +.. NOTE:: + In order to create events in a group other than ``Test``, the user must + be explicitly authorized. Thus, events in the ``CBC`` or ``Burst`` groups, + for example, can only be created by authorized users. For more information + on authorization, see :ref:`auth`. + +Now suppose that a subsequent analysis has updated the values in the original +``coinc.xml`` file. We can *replace* the original event since we know the +``graceid``:: + + new_event_file = "/path/to/new_coinc.xml" + r = client.replaceEvent(graceid, new_event_file) + +This has the effect of updating the values of the various database fields, but +the original event file is kept and can be found in the full event log. + +Annotating events +====================================== + +As discussed in :ref:`annotation_models`, "annotations" refer to pieces of +information added to an event after the time of its creation. Most commonly, +these take the form of event log messages or electromagnetic observation +records (see :ref:`create_emobservation`). The following demonstrates how to +add a log message to an existing event:: + + from ligo.gracedb.rest import GraceDb + client = GraceDb() + + graceid = 'T160779' + message = 'This is a test of the emergency commenting system.' + filename = '/path/to/my_plot.png' + + r = client.writeLog(graceid, message, filename, tagname='analyst_comments') + +In this example, a plot is uploaded to the log messages of event +``T160779`` with a text comment. The new log message is also tagged with +the ``analyst_comments`` tag, which displays the plot and comment in a +special section for comments from (human) data analysts. The tag name +can actually be anything the user wants, but only a limited list of tags +are *blessed* in the sense that they affect the display of information. +For a reminder of which tags are blessed see :ref:`annotation_models`. + +Tags can also be added and removed from existing annotations. See the +docstrings for the client methods ``createTag`` and ``deleteTag``. + +Applying labels +==================================== + +To add a label to an event, the ``graceid`` of the event and the name +of the label must be known:: + + from ligo.gracedb.rest import GraceDb + client = GraceDb() + + graceid = 'T160779' + label_name = 'DQV' # meaning data quality veto + + r = client.writeLabel(graceid, label_name) + +Care should be taken when applying labels to non-test events, since this +affects the sending of alerts related tor potential electromagnetic followup. + +.. _command_line_client: + +Using the command-line client +===================================== + +The GraceDB command-line client, ``gracedb``, is essentially a thin +wrapper on the Python client. The examples above could be repeated +with the command-line client as follows (assuming ``bash``):: + + # Create the new event and store the uid + GRACEID="$(gracedb Test gstlal LowMass /path/to/coinc.xml)" + + # Replace the event + gracedb replace $GRACEID /path/to/new_coinc.xml + + # Annotate an event with a plot + COMMENT="This is a test of the emergency commenting system." + gracedb --tag-name=analyst_comments upload T160779 /path/to/my_plot.png $COMMENT + + # Label an event + gracedb label T160779 DQV + +Type ``gracedb -h`` for detailed help with the command-line client. + +.. _coding_against_api: + +Coding against the GraceDB REST API +======================================================= + +Some users may wish to code directly against the GraceDB REST API rather +than use the Python or command-line clients. In order to do this, the user +will need to know which resources are exposed by which URLS, and which HTTP +methods those URLs allow. Fortunately, the +`Django REST Framework <http://www.django-rest-framework.org>`_ (on which +the GraceDB API is built) provides +a convenient *browseable* version of the API which serves as a reference. +The root of the API can be found here: + +`https://gracedb.ligo.org/apiweb/ <https://gracedb.ligo.org/apiweb/>`_ + +A glance at the upper-right hand corner shows that this URL supports only +``OPTIONS`` and ``GET``. The body is a collection of JSON information provided +by the root resource, including ``links``. One of these links points to the +event list resource: + +`https://gracedb.ligo.org/apiweb/events/ <https://gracedb.ligo.org/apiweb/events/>`_ + +which also supports ``POST`` (see the bottom of the page). New events are +created by ``POST``-ing to the event list resource. This results in a new +event with a unique URL. If the parameters of the event change, the event +can be replaced by a ``PUT`` request to the event URL. In a similar manner, +new log messages are created by ``POST``-ing to the event log list associated +with a particular event. The data expected by these target URLs is not yet +documented here. However, the source code of the GraceDB Python client +can be consulted for examples. + + +.. include info about coding directly against the REST API here. diff --git a/doc/source/web.rst b/doc/source/web.rst new file mode 100644 index 0000000000000000000000000000000000000000..26b9b6939ab43a8b2f9f60958673ee99edeeb225 --- /dev/null +++ b/doc/source/web.rst @@ -0,0 +1,25 @@ +========================== +Using the web interface +========================== + +Searching for events +========================== + +Under construction. + +Understanding the event page +=============================== + +.. links to Roy's videos + +.. include a bit on tagging annotations + +Under construction. + +Signing up for email alerts (LVC only) +======================================================= + +Under construction. + +.. Why only for LVC users? Because we have GCN for LV-EM users and (eventually) the general public. + diff --git a/gracedb/views.py b/gracedb/views.py index b26e396e496e6e2da3769ffaf58748e948ceae3c..6285d8719c4283b27dd11b0cff392ce98c9ae220 100644 --- a/gracedb/views.py +++ b/gracedb/views.py @@ -72,6 +72,9 @@ def index(request): {}, context_instance=RequestContext(request)) +def navbar_only(request): + return render_to_response('navbar_only.html', {}, context_instance=RequestContext(request)) + # SP Info and Privacy pages are required for Federation with InCommon. def spinfo(request): return render_to_response('gracedb/spinfo.html', {}, context_instance=RequestContext(request)) diff --git a/templates/base.html b/templates/base.html index 9ce40ec4f200aeab733ac54a537487e75fc915cf..9a3ab0a2898de17570f0c70ea5d66b99be702c72 100644 --- a/templates/base.html +++ b/templates/base.html @@ -44,6 +44,7 @@ function changeTime(obj, label) { {# if user_is_internal #} <li id="nav-userprofile"><a href="{% url "userprofile-home" %}">Options</a></li> {# endif #} + <li id="nav-docs"><a href="{% url "home" %}documentation/">Documentation</a></li> {% if user %} {% if user.first_name %} <li id="nav-user">Authenticated as: {{ user.first_name }} {{user.last_name }}</li> diff --git a/templates/navbar_only.html b/templates/navbar_only.html new file mode 100644 index 0000000000000000000000000000000000000000..c9457388a72537af78c76e3a5a15f7010ebf44ff --- /dev/null +++ b/templates/navbar_only.html @@ -0,0 +1,53 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +{% block headcontents %} + <link rel="stylesheet" href="{{STATIC_URL}}css/style.css" /> +{% endblock %} +</head> +<div id="content"> + +<center> +<h1 class="docnav"> GraceDB — Gravitational Wave Candidate Event Database</h1> +</center> + +{% block nav %} +<ul id="nav"> + <li id="nav-home"><a href="{% url "home" %}">Home</a></li> + <li id="nav-search"><a href="{% url "search" %}">Search</a></li> + {% if user_is_internal %} + <li id="nav-create"><a href="{% url "create" %}">Create</a></li> + <li id="nav-reports"><a href="{% url "reports" %}">Reports</a></li> + <li id="nav-feeds"><a href="{% url "feeds" %}">RSS</a></li> + {% endif %} + <li id="nav-latest"><a href="{% url "latest" %}">Latest</a></li> + {# if user_is_internal #} + <li id="nav-userprofile"><a href="{% url "userprofile-home" %}">Options</a></li> + {# endif #} + <li id="nav-docs"><a href="{% url "home" %}documentation/">Documentation</a></li> + {% if user %} + {% if user.first_name %} + <li id="nav-user">Authenticated as: {{ user.first_name }} {{user.last_name }}</li> + {% else %} + <li id="nav-user">Authenticated as: {{ user.username }}</li> + {% endif %} + {% else %} + <li id="nav-login"><a href="{{ login_url }}">Login</a></li> + {% endif %} +</ul> +<center> + {% if config_name %} + <h1 class="docnav" style="color: red;"> + {{config_name}} + </h1> + {% endif %} +</center> +{% endblock %} + + <p> </p> <!-- bad way to create vertical space --> + +</div> + +</body> +</html> diff --git a/urls.py b/urls.py index 608640d421659829d6f2aafa03eb6c809dfd0c39..8ddd135853de5732a30e05dcc9677bad0f23ce9d 100644 --- a/urls.py +++ b/urls.py @@ -17,6 +17,7 @@ feeds = { urlpatterns = patterns('', url (r'^$', 'gracedb.views.index', name="home"), + url (r'^navbar_only$', 'gracedb.views.navbar_only', name="navbar-only"), url (r'^SPInfo', 'gracedb.views.spinfo', name="spinfo"), url (r'^SPPrivacy', 'gracedb.views.spprivacy', name="spprivacy"), url (r'^DiscoveryService', 'gracedb.views.discovery', name="discovery"),