Commit a0ed2490 authored by James Clark's avatar James Clark
Browse files

Merge branch 'master' of git.ligo.org:james-clark/gwrucio

parents 2fc19287 28f9de48
Pipeline #78967 passed with stages
in 1 minute and 6 seconds
......@@ -43,7 +43,7 @@ RUN pip install --no-cache-dir \
# Directory setup for rucio configuration and data directory bindings
RUN mkdir -p /opt/rucio/etc /archive /net /hdfs /local
COPY configs/client/etc/rucio.cfg /opt/rucio/etc/rucio.cfg
#COPY configs/client/etc/rucio.cfg /opt/rucio/etc/rucio.cfg
# --------- Customisation & Conveniences --------- #
COPY entrypoint/ps1.sh /etc/profile.d/
......
......@@ -117,18 +117,6 @@ def get_parser():
action="store_true",
help="""Print all logging info""")
oparser.add_argument("--lifetime",
type=float,
default=None,
required=False,
help="""Dataset lifetime in seconds""")
oparser.add_argument("--force-checksums",
default=False,
action="store_true",
help="""Compute checksums and
register files even if they are already present""")
subparsers = oparser.add_subparsers()
#
......@@ -324,6 +312,7 @@ def daemon(aparser):
aparser.cachefile)
# If necessary, wait for diskcache file to appear
# FIXME: add exception for bad diskcache (i.e., retry if exists but is empty/incomplete)
# FIXME: update minimum_gps to time of last check, so we don't waste
# time checking files we just registered
while True:
......
# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2018
# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012, 2015-2016
# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013
# - Nicolo Magini, <nicolo.magini@cern.ch>, 2018
[common]
logdir = /var/log/rucio
loglevel = DEBUG
mailtemplatedir=/opt/rucio/etc/mail_templates
[client]
rucio_host = https://rucio-dev.ligo.caltech.edu:443
auth_host = https://rucio-dev.ligo.caltech.edu:443
;auth_type = userpass
#auth_type = gss
rucio_host = https://ligo-rucio.nautilus.optiputer.net
auth_host = https://ligo-rucio.nautilus.optiputer.net
;auth_type = x509
auth_type = x509_proxy
#auth_type = ssh
username = ddmlab
password = secret
;ca_cert = /etc/grid-security/ligo-chain.crt
client_cert = /opt/x509/hostcert.pem
client_key = /opt/x509/hostkey.pem
ca_cert = /etc/grid-security/certificates
client_cert = /root/.globus/usercert.pem
client_key = /root/.globus/userkey.pem
client_x509_proxy = $X509_USER_PROXY
ssh_private_key = $HOME/.ssh/id_rsa
auth_type = userpass
username = ligolab
password = XXXXXXXXXXXXXX
account = root
request_retries = 3
[database]
default = mysql://rucio:rucio@rucio-db/rucio
pool_recycle=3600
echo=0
pool_reset_on_return=rollback
[policy]
permission = generic
schema = generic
......@@ -41,16 +28,64 @@ lfn2pfn_algorithm_default = ligo
support = james.clark@ligo.org
support_rucio = https://github.com/rucio/rucio/issues/
[database]
#default = sqlite:////tmp/rucio.db
#default =
oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
#default =
oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
#schema=atlas_rucio # only for cern oracle
#default = mysql://rucio:rucio@localhost/rucio
default = postgresql://rucio:rucio@rucio-dev.ligo.caltech.edu/rucio
pool_recycle=3600
echo=0
pool_reset_on_return=rollback
[bootstrap]
; Used to create root account
x509_identity = /DC=org/DC=incommon/C=US/ST=CA/L=Pasadena/O=California Institute of Technology/OU=Laser Interferometer Gravitational-Wave Observatory/CN=rucio.ligo.caltech.edu
x509_email = james.clark@ligo.org
userpass_identity = ligolab
userpass_pwd = XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
userpass_email = james.clark@ligo.org
gss_identity=None
gss_email=None
ssh_identity=None
ssh_email=None
[conveyor]
scheme = srm,gsiftp
transfertool = fts3
ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
cacert = /etc/grid-security/certificates
usercert = /opt/rucio/web/x509up
cachedir = /opt/rucio/cache
;default-source-strategy=orderly
; scheme = srm,gsiftp,root,https,davs
; transfertool = fts3
; cache_time = 600
; use_deterministic_id = True
; poll_timeout = 30
; submit_timeout = 30
; bring_online = 604800
; queue_mode = strict
; cacert = /etc/pki/tls/certs/CERN-bundle.pem
; usercert = /opt/rucio/etc/ddmadmin_proxy/x509up
; ftshosts = https://lcgfts3.gridpp.rl.ac.uk:8446,https://fts3-pilot.cern.ch:8446,https://fts.usatlas.bnl.gov:8446,https://fts3-test.gridpp.rl.ac.uk:8446,https://fts3-atlas.cern.ch:8446,https://fts3-devel.cern.ch:8446
; using_memcache = True
; ftsmonhosts = https://lcgfts3.gridpp.rl.ac.uk:8449,https://fts3-pilot.cern.ch:8449,https://fts.usatlas.bnl.gov:8449,https://fts3-test.gridpp.rl.ac.uk:8449,https://fts3-atlas.cern.ch:8449,https://fts3-atlas.cern.ch:8449,https://fts3-devel.cern.ch:8449
;
[alembic]
cfg = /opt/rucio/etc/alembic.ini
[messaging-fts3]
port = 61613
ssl_key_file = /opt/x509/hostkey.pem
ssl_cert_file = /opt/x509/hostcert.pem
use_ssl = false
destination = /topic/transfer.fts_monitoring_queue_state
brokers = rucio.ligo.caltech.edu
voname = ligo
[messaging-hermes]
username = admin
password = secret
port = 61613
nonssl_port = 61613
use_ssl = false
ssl_key_file = /opt/x509/hostkey.pem
ssl_cert_file = /opt/x509/hostcert.pem
destination = /topic/rucio.events
brokers = rucio.ligo.caltech.edu
voname = ligo
email_from = Rucio <james.clark@ligo.org>
email_test = james.clark@ligo.org
#FROM ligo/base:el7
#FROM containers.ligo.org/lscsoft/lalsuite/lalsuite-v6.49a:stretch
FROM sl:7
ARG version
MAINTAINER James Alexander Clark <james.clark@ligo.org>
RUN echo "Building rucio server"
## RHEL/CentOS 7 64-Bit, OSG & postgresql-10 ##
RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
rpm -ivh http://repo.opensciencegrid.org/osg/3.4/osg-3.4-el7-release-latest.rpm && \
rpm -Uvh https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-redhat10-10-2.noarch.rpm
run yum update -y && \
yum install -y git \
vim \
bzip2 \
kernel-devel \
gcc \
make \
perl \
python-devel \
python-psycopg2 \
osg-pki-tools \
fetch-crl \
memcached \
postgresql10-server \
postgresql10 \
gfal2 \
gfal2-python \
gfal2-util \
gfal2-all \
globus-proxy-utils \
voms-clients-cpp \
httpd-devel \
openssl-devel \
mod_wsgi \
mod_auth_kerb \
gridsite && \
yum clean all && \
rm -rf /var/cache/yum
#
# pip
#
RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
python get-pip.py && \
rm get-pip.py
RUN curl -O https://raw.githubusercontent.com/astroclark/rucio/ligo_lfn2pfn/requirements.readthedocs.txt && \
pip install -r requirements.readthedocs.txt supervisor
RUN pip install --no-cache-dir git+https://github.com/astroclark/rucio.git@ligo_lfn2pfn supervisor
#COPY configs/server/rucio.cfg /opt/rucio/etc/rucio.cfg
#
#
# gwrucio python requirements
# RUN pip install --upgrade pip setuptools
#RUN rm -rf /usr/lib/python2.7/site-packages/ipaddress*
# COPY requirements.txt /tmp
# RUN pip install --upgrade -r requirements.txt
# RUN rm requirements.txt
#
# gwrucio components
# COPY /bin /tmp/bin
# COPY /gwrucio /tmp/gwrucio
# COPY /setup.py /tmp/setup.py
# COPY /README.md /tmp/README.md
# RUN python setup.py install
# RUN rm -rf /tmp/*
#
# RUN mkdir /archive /net /hdfs
#
# rucio configuration
# COPY /configs/client/etc/rucio.cfg /opt/rucio/etc/rucio.cfg
#
#COPY /entrypoint/startup /usr/local/bin/startup
#ENTRYPOINT [ "/usr/local/bin/startup" ]
#CMD ["/bin/bash"]
#ENTRYPOINT ["/bin/bash"]
......@@ -132,11 +132,11 @@ class DatasetInjector(object):
files = data['filelist'][:]
# Get dictionary of files and metadata to register
self.logger.info("Checking for pre-registered files")
files = self._reduce_file_list(files)
if not files:
self.logger.info("All requested files report a replica at %s",
self.logger.info("No new replicas to add at %s",
self.rse_info['rse'])
self.files = []
else:
self._enumerate_uploads(files)
......@@ -147,7 +147,7 @@ class DatasetInjector(object):
logger = self.logger
try:
# Add rule in here if DID does not exist
logger.debug("Trying to create dataset: %s", self.dataset_name)
logger.info("Trying to create dataset: %s", self.dataset_name)
self.client.add_dataset(scope=self.scope,
name=self.dataset_name,
rules=[{
......@@ -175,6 +175,7 @@ class DatasetInjector(object):
logger = self.logger
# Eliminate invalid files from a copy of the list for easier logging
reduced_files = files[:]
logger.info("%d files in list", len(files))
for fil in files:
# Check file exists and is a file
......@@ -382,8 +383,8 @@ class DatasetInjector(object):
else:
# Register replica
logger.info("File %s already exits at RSE",
filemd['did_name'])
logger.debug("File %s already exits at RSE",
filemd['did_name'])
replica_for_api = convert_file_for_api(filemd)
if self.client.add_replicas(rse=self.rse_info['rse'],
......
H-H1_HOFT_C00:
scope: "ER13"
regexp: "H-H1_HOFT_C00"
minimum-gps: 1228838418
maximum-gps: 2000000000
rse: LIGO-WA-ARCHIVE
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment