Commit 4ece7675 authored by James Clark's avatar James Clark
Browse files

make upload check optional

parent fd7d1fab
......@@ -9,9 +9,7 @@ RUN curl -o lscsoft-production-config-1.3-1.el7.noarch.rpm http://software.ligo.
rm lscsoft-production-config-1.3-1.el7.noarch.rpm
RUN yum install -y epel-release.noarch \
yum upgrade -y && \
yum clean all && \
rm -rf /var/cache/yum
yum upgrade -y
RUN yum install -y \
git \
......@@ -23,7 +21,9 @@ RUN yum install -y \
openssl-devel \
python-devel \
python-pip \
vim
vim \
vo-client \
voms-clients-cpp
RUN yum install -y \
gfal2 \
......
......@@ -124,6 +124,12 @@ def get_parser():
help="""YAML instructions
describing dataset""")
oparser.add_argument('-u',
"--allow-uploads",
default=False,
required=False,
help="""Test for file existence and attempt uploads""")
oparser.add_argument("--dry-run",
default=False,
action="store_true",
......@@ -286,12 +292,12 @@ def list_file_infos(filename):
"""
Read a CSV file listing filenames, checksums and sizes into a dictionary of
file attributes
:param filename: path to CSV file with filepath, bytes, adler32, md5
:returns: a dictionary of dictionaries `{bytes, adler32, md5}`, keyed by
filenames
"""
fileinfos=dict()
fileinfos = dict()
with open(filename, 'r') as finfo:
for line in finfo.readlines():
elem = line.split()
......@@ -428,7 +434,7 @@ def daemon(aparser):
return SUCCESS
def inject_data(dataset_name, rset, rse_info, dry_run=False):
def inject_data(dataset_name, rset, rse_info, allow_uploads=False, dry_run=False):
"""
Execute the main registration loop
......@@ -441,6 +447,7 @@ def inject_data(dataset_name, rset, rse_info, dry_run=False):
rse_info,
dataset_name,
data=rset,
allow_uploads=allow_uploads,
logger=LOGGER)
if dry_run:
......
......@@ -98,8 +98,9 @@ class DatasetInjector(object):
dictionary including a diskcache object. If the latter, DatasetInjector
will parse out the file list and metadata.
"""
# pylint: disable=too-many-instance-attributes,too-many-arguments
def __init__(self, rse_info, dataset_name, data, logger=None):
def __init__(self, rse_info, dataset_name, data, allow_uploads=False, logger=None):
if not logger:
logger = logging.getLogger('%s.null' % __name__)
......@@ -122,6 +123,7 @@ class DatasetInjector(object):
self.lifetime = None
self.rse_info = rse_info
self.allow_uploads = allow_uploads
self.client = Client(rucio_host=config_get('client', 'rucio_host'))
# Read and attach list of file attributes (filename, adler32, md5, bytes)
......@@ -131,7 +133,6 @@ class DatasetInjector(object):
self.logger.info("Using pre-computed file sizes and checksums")
except KeyError:
self.logger.info("Computing file sizes and checksums on the fly")
pass
# Support lists OR diskcache for files
try:
......@@ -386,8 +387,12 @@ class DatasetInjector(object):
# Register files and attach to dataset
for filemd in self.files:
logger.info("Adding %s to catalog", filemd['did_name'])
# Test for file existence at end-point
file_exists = rsemgr.exists(self.rse_info, filemd['pfn'])
if self.allow_uploads:
# Test for file existence at end-point
file_exists = rsemgr.exists(self.rse_info, filemd['pfn'])
else:
file_exists = True
if not self._check_replica(filemd['did_name']) or not file_exists:
# IF not in catalog OR doesn't exist on the RSE, test for
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment