Commit 728774c1 authored by James Clark's avatar James Clark
Browse files

file characteristics from file working

parent 8cb5655f
Pipeline #83157 passed with stages
in 57 seconds
......@@ -178,6 +178,12 @@ Note: register a list of files from a text file:
help="""Files for
registration""")
add_files_parser.add_argument("--file-infos",
type=str,
default=None,
required=False,
help="""File list with bytes, adler32, md5""")
add_files_parser.add_argument("--rset",
type=str,
default=None,
......@@ -289,7 +295,7 @@ def list_file_infos(filename):
with open(filename, 'r') as finfo:
for line in finfo.readlines():
elem = line.split()
fileinfos[elem[0]] = {'bytes':elem[1], 'adler32':elem[2], 'md5':elem[3]}
fileinfos[elem[0]] = {'bytes':int(elem[1]), 'adler32':elem[2], 'md5':elem[3]}
return fileinfos
......@@ -316,9 +322,9 @@ def add_files(aparser):
try:
# Add a file-info file:
rset['fileinfos'] = list_file_infos(aparser.files_infos)
rset['fileinfos'] = list_file_infos(aparser.file_infos)
except AttributeError:
continue
pass
inject_data(aparser.rset, rset, rse_info, aparser.dry_run)
......
......@@ -128,8 +128,10 @@ class DatasetInjector(object):
# If file is not found, continue to compute attributes on the fly
try:
self.fileinfos = data['fileinfos']
self.logger.info("Using pre-computed file sizes and checksums")
except KeyError:
continue
self.logger.info("Computing file sizes and checksums on the fly")
pass
# Support lists OR diskcache for files
try:
......@@ -333,11 +335,12 @@ class DatasetInjector(object):
new_item['dirname'] = os.path.dirname(filepath)
new_item['basename'] = os.path.basename(filepath)
# Try getting file info from fileinfos dict
try:
# Try getting file info from fileinfos dict
new_item['bytes'] = self.fileinfos[new_item['basename']]['bytes']
new_item['adler32'] = self.fileinfos[new_item['basename']]['adler32']
new_item['md5'] = self.fileinfos[new_item['basename']]['md5']
new_item['bytes'] = self.fileinfos[new_item['path']]['bytes']
new_item['adler32'] = self.fileinfos[new_item['path']]['adler32']
new_item['md5'] = self.fileinfos[new_item['path']]['md5']
except AttributeError:
# Compute file infos on the fly
new_item['bytes'] = os.stat(filepath).st_size
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment