Commit 29e9785d authored by James Clark's avatar James Clark
Browse files

Updated dependencies based on rucio daemons container

parent 2d016f07
Pipeline #81145 passed with stages
in 58 seconds
......@@ -5,45 +5,52 @@ RUN echo "Building gwrucio"
RUN curl -o lscsoft-production-config-1.3-1.el7.noarch.rpm http://software.ligo.org/lscsoft/scientific/7/x86_64/production/l/lscsoft-production-config-1.3-1.el7.noarch.rpm && \
rpm -ivh lscsoft-production-config-1.3-1.el7.noarch.rpm && \
rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
rpm -ivh http://repo.opensciencegrid.org/osg/3.4/osg-3.4-el7-release-latest.rpm && \
rm lscsoft-production-config-1.3-1.el7.noarch.rpm
RUN yum upgrade -y && \
RUN yum install -y epel-release.noarch \
yum upgrade -y && \
yum clean all && \
rm -rf /var/cache/yum
RUN yum install -y git \
vim \
RUN yum install -y \
git \
gcc \
make \
MySQL-python \
python-devel \
python-psycopg2 \
openssl-devel \
gfal2-all \
gfal2-util \
ldg-client \
ldas-tools-diskcacheAPI \
libaio \
MySQL-python \
openssl-devel \
python-devel \
python-pip \
vim \
yum clean all && \
rm -rf /var/cache/yum
RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
python get-pip.py && \
rm get-pip.py
RUN yum install -y \
gfal2 \
gfal2-plugin-file \
gfal2-plugin-gridftp \
gfal2-plugin-http \
gfal2-plugin-srm \
gfal2-plugin-xrootd \
gfal2-python \
gfal2-util \
xrootd-client
RUN pip install --upgrade pip setuptools
RUN rm -rf /usr/lib/python2.7/site-packages/ipaddress*
RUN pip install --no-cache-dir \
rucio[mysql] \
git+https://git.ligo.org/james-clark/gwrucio.git#egg=gwrucio
j2cli \
lalsuite \
lscsoft-glue \
psycopg2-binary \
PyYAML \
flake8 \
pylint \
git+https://git.ligo.org/james-clark/gwrucio.git#egg=gwrucio
rucio[postgresql]
# Directory setup for rucio configuration and data directory bindings
RUN mkdir -p /opt/rucio/etc /archive /net /hdfs /local
#COPY configs/client/etc/rucio.cfg /opt/rucio/etc/rucio.cfg
# --------- Customisation & Conveniences --------- #
COPY entrypoint/ps1.sh /etc/profile.d/
......
[common]
logdir = /var/log/rucio
loglevel = DEBUG
[client]
rucio_host = https://ligo-rucio.nautilus.optiputer.net
auth_host = https://ligo-rucio.nautilus.optiputer.net
;auth_type = x509
client_cert = /opt/x509/hostcert.pem
client_key = /opt/x509/hostkey.pem
ca_cert = /etc/grid-security/certificates
auth_type = userpass
username = ligolab
password = XXXXXXXXXXXXXX
account = root
request_retries = 3
[database]
default = mysql://rucio:rucio@rucio-db/rucio
pool_recycle=3600
echo=0
pool_reset_on_return=rollback
[policy]
permission = generic
schema = generic
lfn2pfn_algorithm_default = ligo
support = james.clark@ligo.org
support_rucio = https://github.com/rucio/rucio/issues/
[bootstrap]
; Used to create root account
x509_identity = /DC=org/DC=incommon/C=US/ST=CA/L=Pasadena/O=California Institute of Technology/OU=Laser Interferometer Gravitational-Wave Observatory/CN=rucio.ligo.caltech.edu
x509_email = james.clark@ligo.org
userpass_identity = ligolab
userpass_pwd = XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
userpass_email = james.clark@ligo.org
gss_identity=None
gss_email=None
ssh_identity=None
ssh_email=None
[conveyor]
scheme = srm,gsiftp
transfertool = fts3
ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
cacert = /etc/grid-security/certificates
usercert = /opt/rucio/web/x509up
cachedir = /opt/rucio/cache
;default-source-strategy=orderly
; scheme = srm,gsiftp,root,https,davs
; transfertool = fts3
; cache_time = 600
; use_deterministic_id = True
; poll_timeout = 30
; submit_timeout = 30
; bring_online = 604800
; queue_mode = strict
; cacert = /etc/pki/tls/certs/CERN-bundle.pem
; usercert = /opt/rucio/etc/ddmadmin_proxy/x509up
; ftshosts = https://lcgfts3.gridpp.rl.ac.uk:8446,https://fts3-pilot.cern.ch:8446,https://fts.usatlas.bnl.gov:8446,https://fts3-test.gridpp.rl.ac.uk:8446,https://fts3-atlas.cern.ch:8446,https://fts3-devel.cern.ch:8446
; using_memcache = True
; ftsmonhosts = https://lcgfts3.gridpp.rl.ac.uk:8449,https://fts3-pilot.cern.ch:8449,https://fts.usatlas.bnl.gov:8449,https://fts3-test.gridpp.rl.ac.uk:8449,https://fts3-atlas.cern.ch:8449,https://fts3-atlas.cern.ch:8449,https://fts3-devel.cern.ch:8449
;
[alembic]
cfg = /opt/rucio/etc/alembic.ini
[messaging-fts3]
port = 61613
ssl_key_file = /opt/x509/hostkey.pem
ssl_cert_file = /opt/x509/hostcert.pem
use_ssl = false
destination = /topic/transfer.fts_monitoring_queue_state
brokers = rucio.ligo.caltech.edu
voname = ligo
[messaging-hermes]
username = admin
password = secret
port = 61613
nonssl_port = 61613
use_ssl = false
ssl_key_file = /opt/x509/hostkey.pem
ssl_cert_file = /opt/x509/hostcert.pem
destination = /topic/rucio.events
brokers = rucio.ligo.caltech.edu
voname = ligo
email_from = Rucio <james.clark@ligo.org>
email_test = james.clark@ligo.org
# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Vincent Garonne, <vincent.garonne@cern.ch>, 2014
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = /opt/rucio/lib/rucio/db/sqla/migrate_repo
#script_location = /usr/lib/python2.7/site-packages/rucio/db/sqla/migrate_repo/
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
#sqlalchemy.url = mysql://rucio:rucio@localhost/rucio
sqlalchemy.url = postgresql://rucio:rucio@localhost/rucio
# version_table_schema = ATLAS_RUCIO
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
<?xml version="1.0"?>
<gacl>
<entry>
<any-user/>
<allow><read/><list/></allow>
</entry>
</gacl>
# PostgreSQL Client Authentication Configuration File
# ===================================================
#
# Refer to the "Client Authentication" section in the PostgreSQL
# documentation for a complete description of this file. A short
# synopsis follows.
#
# This file controls: which hosts are allowed to connect, how clients
# are authenticated, which PostgreSQL user names they can use, which
# databases they can access. Records take one of these forms:
#
# local DATABASE USER METHOD [OPTIONS]
# host DATABASE USER ADDRESS METHOD [OPTIONS]
# hostssl DATABASE USER ADDRESS METHOD [OPTIONS]
# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS]
#
# (The uppercase items must be replaced by actual values.)
#
# The first field is the connection type: "local" is a Unix-domain
# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
# plain TCP/IP socket.
#
# DATABASE can be "all", "sameuser", "samerole", "replication", a
# database name, or a comma-separated list thereof. The "all"
# keyword does not match "replication". Access to replication
# must be enabled in a separate record (see example below).
#
# USER can be "all", a user name, a group name prefixed with "+", or a
# comma-separated list thereof. In both the DATABASE and USER fields
# you can also write a file name prefixed with "@" to include names
# from a separate file.
#
# ADDRESS specifies the set of hosts the record matches. It can be a
# host name, or it is made up of an IP address and a CIDR mask that is
# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
# specifies the number of significant bits in the mask. A host name
# that starts with a dot (.) matches a suffix of the actual host name.
# Alternatively, you can write an IP address and netmask in separate
# columns to specify the set of hosts. Instead of a CIDR-address, you
# can write "samehost" to match any of the server's own IP addresses,
# or "samenet" to match any address in any subnet that the server is
# directly connected to.
#
# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
# Note that "password" sends passwords in clear text; "md5" or
# "scram-sha-256" are preferred since they send encrypted passwords.
#
# OPTIONS are a set of options for the authentication in the format
# NAME=VALUE. The available options depend on the different
# authentication methods -- refer to the "Client Authentication"
# section in the documentation for a list of which options are
# available for which authentication methods.
#
# Database and user names containing spaces, commas, quotes and other
# special characters must be quoted. Quoting one of the keywords
# "all", "sameuser", "samerole" or "replication" makes the name lose
# its special character, and just match a database or username with
# that name.
#
# This file is read on server startup and when the server receives a
# SIGHUP signal. If you edit the file on a running system, you have to
# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
# or execute "SELECT pg_reload_conf()".
#
# Put your actual configuration here
# ----------------------------------
#
# If you want to allow non-local connections, you need to add more
# "host" records. In that case you will also need to make PostgreSQL
# listen on a non-local interface via the listen_addresses
# configuration parameter, or via the -i or -h command line switches.
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all all md5
# IPv4 local connections:
host all all 127.0.0.1/32 md5
# IPv6 local connections:
host all all ::1/128 md5
# Allow replication connections from localhost, by a user with tmd5
# replication privilege.
local replication all md5
host replication all 127.0.0.1/32 md5
host replication all ::1/128 md5
host all all 0.0.0.0/0 md5
# PostgreSQL Client Authentication Configuration File
# ===================================================
#
# Refer to the "Client Authentication" section in the PostgreSQL
# documentation for a complete description of this file. A short
# synopsis follows.
#
# This file controls: which hosts are allowed to connect, how clients
# are authenticated, which PostgreSQL user names they can use, which
# databases they can access. Records take one of these forms:
#
# local DATABASE USER METHOD [OPTIONS]
# host DATABASE USER ADDRESS METHOD [OPTIONS]
# hostssl DATABASE USER ADDRESS METHOD [OPTIONS]
# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS]
#
# (The uppercase items must be replaced by actual values.)
#
# The first field is the connection type: "local" is a Unix-domain
# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
# plain TCP/IP socket.
#
# DATABASE can be "all", "sameuser", "samerole", "replication", a
# database name, or a comma-separated list thereof. The "all"
# keyword does not match "replication". Access to replication
# must be enabled in a separate record (see example below).
#
# USER can be "all", a user name, a group name prefixed with "+", or a
# comma-separated list thereof. In both the DATABASE and USER fields
# you can also write a file name prefixed with "@" to include names
# from a separate file.
#
# ADDRESS specifies the set of hosts the record matches. It can be a
# host name, or it is made up of an IP address and a CIDR mask that is
# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
# specifies the number of significant bits in the mask. A host name
# that starts with a dot (.) matches a suffix of the actual host name.
# Alternatively, you can write an IP address and netmask in separate
# columns to specify the set of hosts. Instead of a CIDR-address, you
# can write "samehost" to match any of the server's own IP addresses,
# or "samenet" to match any address in any subnet that the server is
# directly connected to.
#
# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256",
# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert".
# Note that "password" sends passwords in clear text; "md5" or
# "scram-sha-256" are preferred since they send encrypted passwords.
#
# OPTIONS are a set of options for the authentication in the format
# NAME=VALUE. The available options depend on the different
# authentication methods -- refer to the "Client Authentication"
# section in the documentation for a list of which options are
# available for which authentication methods.
#
# Database and user names containing spaces, commas, quotes and other
# special characters must be quoted. Quoting one of the keywords
# "all", "sameuser", "samerole" or "replication" makes the name lose
# its special character, and just match a database or username with
# that name.
#
# This file is read on server startup and when the server receives a
# SIGHUP signal. If you edit the file on a running system, you have to
# SIGHUP the server for the changes to take effect, run "pg_ctl reload",
# or execute "SELECT pg_reload_conf()".
#
# Put your actual configuration here
# ----------------------------------
#
# If you want to allow non-local connections, you need to add more
# "host" records. In that case you will also need to make PostgreSQL
# listen on a non-local interface via the listen_addresses
# configuration parameter, or via the -i or -h command line switches.
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 ident
# IPv6 local connections:
host all all ::1/128 ident
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all peer
host replication all 127.0.0.1/32 ident
host replication all ::1/128 ident
This diff is collapsed.
# Core supervisor config is contained in /etc/supervisord.conf
# This file configures the daemons listed at
# https://rucio.readthedocs.io/en/latest/man/daemons.html
;[unix_http_server]
;;file = /tmp/supervisor.sock
;
;[supervisord]
;logfile=/var/log/rucio/supervisord.log
;logfile_maxbytes=1024MB
;logfile_backups=5
;pidfile=/tmp/supervisord.pid
;
;[rpcinterface:supervisor]
;supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
;
;[supervisorctl]
;serverurl=unix:///tmp/supervisor.sock
; [program:rucio-hermes]
; command=/bin/rucio-hermes
; environment=RUCIO_HOME=/opt/rucio/
; childlogdir=/var/log/rucio
; stdout_logfile=/var/log/rucio/hermes.log
; redirect_stderr=true
; autorestart=true
; stopsignal=KILL
; exitcodes=1
; stdout_logfile_maxbytes=50MB
; stdout_logfile_backups=5
;[program:rucio-kronos]
;command=/bin/rucio-kronos
;stdout_logfile=/var/log/rucio/kronos.log
;redirect_stderr=true
;autorestart=true
;stopsignal=KILL
;exitcodes=1
;stdout_logfile_maxbytes=50MB
;stdout_logfile_backups=5
[program:rucio-judge-evaluator]
command=/bin/rucio-judge-evaluator --threads 5
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio
stdout_logfile=/var/log/rucio/judge-evaluator.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-judge-cleaner]
command=/bin/rucio-judge-cleaner --threads 5
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio
stdout_logfile=/var/log/rucio/judge-cleaner.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-judge-repairer]
command=/bin/rucio-judge-repairer --threads 5
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio
stdout_logfile=/var/log/rucio/judge-repairer.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-conveyor-submitter]
command=/bin/rucio-conveyor-submitter --activities "User Subscriptions" --sleep-time 30 --total-threads 5
;command=/bin/rucio-conveyor-submitter --sleep-time 5 --total-threads 10
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/conveyor-transfer-submitter.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-conveyor-poller]
command=/bin/rucio-conveyor-poller
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/conveyor-poller.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-conveyor-finisher]
command=/bin/rucio-conveyor-finisher --activities "User Subscriptions"
environment=RUCIO_HOME=/opt/rucio/
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/conveyor-finisher.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
; [program:rucio-conveyor-receiver]
; command=/bin/rucio-conveyor-receiver --full-mode
; environment=RUCIO_HOME=/opt/rucio/
; childlogdir=/var/log/rucio/
; stdout_logfile=/var/log/rucio/conveyor-receiver.log
; redirect_stderr=true
; autorestart=true
; stopsignal=KILL
; exitcodes=1
; stdout_logfile_maxbytes=50MB
; stdout_logfile_backups=5
;
; [program:rucio-conveyor-throttler]
; command=/bin/rucio-conveyor-throttler
; environment=RUCIO_HOME=/opt/rucio/
; childlogdir=/var/log/rucio/
; stdout_logfile=/var/log/rucio/conveyor-throttler.log
; redirect_stderr=true
; autorestart=true
; stopsignal=KILL
; exitcodes=1
; stdout_logfile_maxbytes=50MB
; stdout_logfile_backups=5
[program:rucio-undertaker]
command=/bin/rucio-undertaker --total-workers 10
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/undertaker.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-reaper]
command=/bin/rucio-reaper --total-workers 10 --greedy --exclude-rses LIGO-CIT-ARCHIVE
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/reaper.log
environment=GLOBUS_THREAD_MODEL=pthread,X509_USER_PROXY=/opt/rucio/etc/web/x509up
;,X509_USER_KEY=/opt/rucio/etc/web/x509up,X509_USER_CERT=/opt/rucio/etc/web/x509up
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-necromancer]
command=/bin/rucio-necromancer
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/necromancer.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-abacus-account]
command=/bin/rucio-abacus-account
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/abacus-account.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
[program:rucio-abacus-rse]
command=/bin/rucio-abacus-rse
childlogdir=/var/log/rucio/
stdout_logfile=/var/log/rucio/abacus-rse.log
redirect_stderr=true
autorestart=true
stopsignal=KILL
exitcodes=1
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
;[program:rucio-transmogrifier]
;command=/bin/rucio-transmogrifier
;childlogdir=/var/log/rucio/
;stdout_logfile=/var/log/rucio/transmogrifier.log