Commit fd7d1fab authored by James Clark's avatar James Clark
Browse files

pipeline for getting checksums

parent 834d3f6e
Pipeline #83164 passed with stages
in 52 seconds
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2017 James Clark <james.clark@ligo.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# pylint: disable=superfluous-parens
"""Comand line tool with HTCondor classes and methods to distribute
gwrucio_registrar file registration tasks in an HTCondor workflow. Creates a
DAG and sub files."""
import sys
import os
import stat
import argparse
from glue import pipeline
SUCCESS = 0
FAILURE = 1
class FileInfoJob(pipeline.CondorDAGJob):
"""
Job class for fileinfos.
"""
def __init__(self, accounting_group="ligo.dev.o3.burst.explore.test",
universe="vanilla"):
self.__executable = "fileinfo.sh"
self.__universe = universe
pipeline.CondorDAGJob.__init__(self, self.__universe, self.__executable)
## Logging info
self.set_stdout_file('logs/fileinfo_$(cluster)-$(process).out')
self.set_stderr_file('logs/fileinfo_$(cluster)-$(process).err')
self.set_log_file('logs/fileinfo_$(cluster)-$(process).log')
try:
os.makedirs('logs')
except OSError:
pass
## Arguments
self.add_condor_cmd('accounting_group', accounting_group)
self.add_condor_cmd('should_transfer_files', 'yes')
## Write the submission file
self.set_sub_file('fileinfos.sub')
class FileInfoNode(pipeline.CondorDAGNode):
"""
DAG node for gwrucio_registrar.
"""
def __init__(self, fileinfo_job):
pipeline.CondorDAGNode.__init__(self, fileinfo_job)
def set_framefile(self, framefile):
"""
Specify the frame to analyse
"""
self.add_var_arg(framefile)
def make_fileinfo():
# pylint: disable=line-too-long
"""
Write a script:
```
#!/bin/bash -e
echo "$1 $(stat -c %s $1 | awk '{print $1}') $(xrdadler32 $1 | awk '{print $1}') $(md5sum $1 | awk '{print $1}')"
```
"""
with open('fileinfo.sh', 'w') as exe:
exe.write("#!/bin/bash -e\n")
exe.write("echo \"$1 $(stat -c %s $1 | awk \'{print $1}\') $(xrdadler32 $1 | awk \'{print $1}\') $(md5sum $1 | awk \'{print $1}\')\"\n")
exestat = os.stat('fileinfo.sh')
os.chmod('fileinfo.sh', exestat.st_mode | stat.S_IEXEC)
def get_parser():
"""
Command line parser
"""
oparser = argparse.ArgumentParser(description=__doc__)
oparser.add_argument("framelist", type=str, default=None,
help="""List of frame paths""")
return oparser
if __name__ == "__main__":
#
# Parse input and choose operation
#
PARSER = get_parser()
#argcomplete.autocomplete(PARSER)
if len(sys.argv) == 1:
PARSER.print_help()
sys.exit(FAILURE)
ARGS = PARSER.parse_args(sys.argv[1:])
with open(ARGS.framelist, 'r') as FRAMESF:
FRAMES = FRAMESF.readlines()
# Work in a directory named after the original reg file
WORKDIR = ARGS.framelist.replace('.txt', '')
try:
os.makedirs(WORKDIR)
except OSError:
print("Failed to create new directory {}. Trying to chdir".format(WORKDIR))
STARTDIR = os.getcwd()
try:
os.chdir(WORKDIR)
except OSError:
print("Failed to chdir to {}, exiting".format(WORKDIR))
sys.exit(FAILURE)
print("Writing workflow in {}".format(WORKDIR))
## Define DAG and job types
DAG = pipeline.CondorDAG(log='fileinfos_pipe.log')
DAG.set_dag_file('fileinfos_pipe')
FILEINFO_JOB = FileInfoJob()
## Define DAG node
FILEINFO_NODE = FileInfoNode(FILEINFO_JOB)
FILEINFO_NODE.set_framefile('/i/am/a/frame')
DAG.add_node(FILEINFO_NODE)
## Write the resulting DAG
DAG.write_sub_files()
DAG.write_dag()
DAG.write_script()
# Move back to start directory
os.chdir(STARTDIR)
print("Complete")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment