moved pysccb to a new repo

parent c05258e4
Pipeline #33799 failed with stage
in 1 minute and 36 seconds
......@@ -5,7 +5,7 @@
- ./*.rpm
- ./*.deb
script:
- ${PYTHON} build-dist.py --build-type ${BUILD_TYPE}
- ${PYTHON} -m pysccb.tools.autobuild --build-type ${BUILD_TYPE}
.build:rpm: &build-rpm
<<: *build
......@@ -17,7 +17,7 @@
yum-utils
rpm-build
git python34-pip python34-jinja2 python34-PyYAML
- ${PYTHON} -m pip install --quiet "GitPython>=2.1.8"
- ${PYTHON} -m pip install --quiet git+https://git.ligo.org/duncanmmacleod/pysccb.git
.build:deb: &build-deb
<<: *build
......@@ -29,7 +29,7 @@
- apt-get -yqq install
dpkg-dev devscripts
git python3-pip python3-git python3-jinja2 python3-yaml
- ${PYTHON} -m pip install --quiet "GitPython>=2.1.8"
- ${PYTHON} -m pip install --quiet git+https://git.ligo.org/duncanmmacleod/pysccb.git
build:el7:
<<: *build-rpm
......
# -*- coding: utf-8 -*-
"""Python helpers for LIGO Software Change Control Board requests
"""
# just in case
import sys
if sys.version_info.major < 3:
raise RuntimeError("pysccb is not compatible with python < 3")
__author__ = 'Duncan Macleod <duncan.macleod@ligo.org>'
__version__ = '0.1.0'
# -*- utf-8 -*-
"""Package build utilities for SCCB
"""
import errno
import glob
import logging
import os
import re
import shutil
import tarfile
import tempfile
from distutils.spawn import find_executable
from .utils import (logged_check_call, logged_check_output)
logger = logging.getLogger('sccb')
logger.setLevel('DEBUG')
rpmbuild_ts_resp = re.compile(r'\AWrote: (?P<srcrpm>\S+)\Z')
# -- utilities ----------------------------------------------------------------
def file_exists(path):
if not os.path.exists(path):
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
def _copy_files(files, outdir):
out = []
for name in files:
target = os.path.join(outdir, os.path.basename(name))
shutil.move(name, target)
out.append(target)
logger.debug('Wrote: {0}'.format(target))
return out
def unpack_tarball(tarball, target=os.path.curdir):
with tarfile.open(tarball, 'r') as tar:
members = tar.getmembers()
first = members[0]
if (not first.isdir() or not
all(m.name.startswith(first.name) for m in members)):
raise RuntimeError("Not all files are in the same directory, "
"bailing out.")
for mbr in members[1:]:
mbr.name = mbr.name.split(os.path.sep, 1)[1]
tar.extractall(path=target, members=members[1:])
return target
# -- RPM ----------------------------------------------------------------------
def _is_spec_file(path):
return path.endswith('.spec')
def _rpmbuild_args(*args, tmpdir=None):
rpmbargs = [find_executable('rpmbuild')]
if tmpdir:
rpmbargs.extend(('--define', '_topdir {0}'.format(tmpdir)))
return rpmbargs + list(args)
def build_src_rpm(tarball_or_spec, *extra_rpmbuild_args, outdir=os.path.curdir):
with tempfile.TemporaryDirectory() as tmpdir:
os.makedirs(os.path.join(tmpdir, 'SPECS'))
# run rpmbuild
rpmcmd = '-bs' if _is_spec_file(tarball_or_spec) else '-ts'
cmd = _rpmbuild_args(*extra_rpmbuild_args + (rpmcmd, tarball_or_spec),
tmpdir=tmpdir)
resp = logged_check_output(cmd).strip().decode('utf-8')
# copy src.rpm into outdir and return path
srcrpm = rpmbuild_ts_resp.match(resp).groupdict()['srcrpm']
return _copy_files([srcrpm], outdir)[0]
def build_binary_rpms(tarball_or_spec, *args, outdir=os.path.curdir,
chroot=False):
if chroot:
return _build_binary_rpms_chroot(tarball_or_spec, *args, outdir=outdir)
return _build_binary_rpms_rpmbuild(tarball_or_spec, *args, outdir=outdir)
def _build_binary_rpms_chroot(tarball_or_spec, *extra_mock_args,
outdir=os.path.curdir):
with tempfile.TemporaryDirectory() as tmpdir:
# generate src rpm
srcrpm = build_src_rpm(tarball_or_spec, outdir=tmpdir)
# run mock to generate binary rpms
cmd = [
find_executable('mock'),
'--resultdir', tmpdir,
'--rebuild',
] + list(extra_mock_args) + [srcrpm]
logged_check_call(cmd)
# copy binary rpms to output
return _copy_files(glob.glob(os.path.join(tmpdir, '*.rpm')), outdir)
def _build_binary_rpms_rpmbuild(tarball_or_spec, *extra_rpmbuild_args,
outdir=os.path.curdir):
spec = _is_spec_file(tarball_or_spec)
with tempfile.TemporaryDirectory() as tmpdir:
os.makedirs(os.path.join(tmpdir, 'SPECS'))
# if installing from tarball, build srcrpm now, so that we can use
# yum-builddep to install BuildRequires
if not spec:
src = build_src_rpm(tarball_or_spec, outdir=tmpdir)
rpmcmd = '--rebuild'
else:
src = tarball_or_spec
rpmcmd = '-bb'
# install BuildRequires using yum-builddep
cmd = [find_executable('yum-builddep'), '-y', '-q', src]
logged_check_call(cmd)
# run rpmbuild
cmd = _rpmbuild_args(*extra_rpmbuild_args + (rpmcmd, src),
tmpdir=tmpdir)
logged_check_call(cmd)
# copy binary rpms to output
return _copy_files(
glob.glob(os.path.join(tmpdir, 'RPMS', '**', '*.rpm')), outdir)
# -- DEB ----------------------------------------------------------------------
def build_debs(tarball, outdir=os.path.curdir):
with tempfile.TemporaryDirectory() as tmpdir:
srcdir = os.path.join(tmpdir, 'src')
# unpack tarball
unpack_tarball(tarball, srcdir)
# create build-deps package
cmd = [find_executable('mk-build-deps'), '--tool', 'apt-get -y',
'--install', '--remove']
logged_check_call(cmd, cwd=srcdir)
# build package
cmd = [find_executable('dpkg-buildpackage'), '-us', '-uc', '-b']
logged_check_call(cmd, cwd=srcdir)
# copy debs to output
return _copy_files(glob.glob(os.path.join(tmpdir, '*.deb')), outdir)
# -*- coding: utf-8 -*-
"""Git interactions for SCCB
"""
import git
def get_changed_files(from_, to_='HEAD'):
repo = git.Repo()
return repo.git.diff('{from_}..{to_}'.format(from_=from_, to_=to_),
name_only=True).splitlines()
# -*- coding: utf-8 -*-
"""Package install utilities for SCCB
"""
import logging
import sys
from distutils.spawn import find_executable
from subprocess import CalledProcessError
from .utils import logged_check_call
def install_yum(*packages):
logged_check_call([find_executable('yum'), 'install'] + list(packages))
def local_install_yum(*packages):
logged_check_call([find_executable('yum'), 'localinstall'] +
list(packages))
def install_deb(*packages):
logged_check_call([find_executable('apt-get'), 'install'] + list(packages))
def local_install_deb(*files):
_install = [find_executable('dpkg'), '--install'] + list(files)
try:
logged_check_call(_install)
except CalledProcessError as exc:
logging.debug('dpkg --install failed, trying something else...')
try:
logged_check_call([find_executable('apt-get'), '-f', 'install'])
logged_check_call(_install)
except CalledProcessError as exc2:
raise exc2 from exc
def install_pip(*packages):
logged_check_call([sys.executable, '-m', 'pip', 'install'] + list(packages))
local_install_pip = install_pip
# -*- coding: utf-8 -*-
"""Package testing utilities
"""
import sys
from distutils.spawn import find_executable
from subprocess import CalledProcessError
from .utils import logged_check_call
def run_test(cmd, **kwargs):
if isinstance(cmd, str):
cmd = cmd.split()
cmd = list(cmd)
exe = cmd[0]
if exe == 'python':
cmd[0] == sys.executable
else:
cmd[0] == find_executable(exe[0])
try:
logged_check_call(cmd, **kwargs)
except CalledProcessError as exc:
raise RuntimeError('Test failed') from exc
# -*- coding: utf-8 -*-
"""Generic utilities
"""
import logging
import os
from functools import wraps
from subprocess import (check_call, check_output)
from urllib.request import urlretrieve
def download_file(remote, target=None):
if target is None:
target = os.path.basename(remote)
target = os.path.abspath(target)
urlretrieve(remote, target)
logging.debug('downloaded {0} to {1}'.format(remote, target))
return target
def _logged_call(func):
@wraps(func)
def decorated(cmd, *args, level=logging.DEBUG, **kwargs):
if not isinstance(level, int):
level = logging.getLevelName(level)
logging.log(level, '$ {0}'.format(' '.join(cmd)))
return func(cmd, *args, **kwargs)
return decorated
logged_check_call = _logged_call(check_call)
logged_check_output = _logged_call(check_output)
# -*- coding: utf-8 -*-
"""YAML configuration utilities
"""
import yaml
from jinja2 import Template
def read_config(source):
"""Read a package configuration from a YAML source file
Parameters
----------
source : `str`, `file`
path to YAML file, or open file object
Returns
-------
data : `object`
the data from the YAML file as parsed by :func:`yaml.load`
"""
if isinstance(source, str):
with open(source, 'rb') as fobj:
return read_config(fobj)
# read file, decode to str (from bytes), render with jinja2, then parse
return yaml.load(Template(source.read().decode('utf-8')).render())
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment