Commit eb97b0d3 authored by Adam Mercer's avatar Adam Mercer
Browse files

Merge branch 'package' into 'master'

Restructure into python package

See merge request adam-mercer/koji-packager!2
parents 9c4e5638 63828205
......@@ -2,3 +2,15 @@
.vscode
*.rpm
*.swp
*.py[cod]
# build files/directories to ignore
/dist/
/build/
/.cache/
/*.egg-info/
/.eggs/
__pycache__/
/.coverage
.pytest_cache/
MANIFEST
stages:
- lint
- dist
- build
- test
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
YUM_OPTS: "-y -q --setopt=cachedir=${CI_PROJECT_DIR}/.cache/yum --setopt=keepcache=1"
cache:
key: "${CI_JOB_NAME}"
paths:
- .cache/pip
.install-epel7: &install-epel7 |
rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
# -- lint -------------------
lint:
stage: lint
image: python:3.7
before_script:
- python -m pip install "flake8>=3.7.0"
script:
- python -m flake8 --output-file "flake8-report.txt"
after_script:
- cat "flake8-report.txt"
- python -m pip install "flake8-junit-report"
- python -m junit_conversor "flake8-report.txt" "junit.xml"
artifacts:
reports:
junit: junit.xml
# -- dist -------------------
dist:tarball:
stage: dist
image: python:3.6
script:
- python -m pip install setuptools
- python setup.py sdist bdist_wheel
artifacts:
expire_in: 18h
paths:
- dist
# -- build ------------------
.yum-install-build-deps: &yum-install-build-deps |
yum ${YUM_OPTS} install yum-utils rpm-build python3-rpm-macros
.build:el: &build-rhel
stage: build
dependencies:
- dist:tarball
script:
# build src rpm
- SRC_RPM=$(rpmbuild -ts dist/gwkoji*.tar.* | cut -d\ -f2)
# install build dependencies for src rpm
- yum-builddep ${YUM_OPTS} ${SRC_RPM}
# build binary rpms
- rpmbuild --rebuild ${SRC_RPM}
# move things into dist/
- DIST_NAME=${CI_JOB_NAME#build:}
- mkdir -pv dist/${DIST_NAME}
- mv -v ~/rpmbuild/RPMS/*/python*-gwkoji-*.rpm dist/${DIST_NAME}/
- rpm -qlp dist/${DIST_NAME}/python*-gwkoji-*.rpm
cache:
key: "${CI_JOB_NAME}"
paths:
- .cache/yum
build:el7:
<<: *build-rhel
image: centos:centos7
before_script:
- *install-epel7
- *yum-install-build-deps
- yum ${YUM_OPTS} install python36
artifacts:
expire_in: 18h
paths:
- dist/el7
# -- test -------------------
.test: &test
stage: test
image: python
dependencies:
- dist:tarball
before_script:
- python -m pip install dist/gwkoji*.tar.*
- python -m pip install
"pytest>=2.8.0"
"pytest-cov"
script:
- ${PYTHON:-python} -m pytest --pyargs gwkoji --cov gwkoji --junitxml ${CI_PROJECT_DIR}/junit.xml
artifacts:
reports:
junit: junit.xml
test:python3.6:
<<: *test
image: python:3.6
test:python3.7:
<<: *test
image: python:3.7
test:el7:
<<: *test
image: centos:centos7
dependencies:
- build:el7
before_script:
- *install-epel7
- yum ${YUM_OPTS} install
python36-pytest
python36-pytest-cov
- yum -y -q --nogpgcheck localinstall dist/el7/python36-gwkoji-*.rpm
- export PYTHON="python3.6"
cache:
key: "${CI_JOB_NAME}"
paths:
- .cache/yum
include COPYING README.md
include gwkoji.spec
Python library for interacting with the Koji RPM build system
%define name gwkoji
%define version 0.1.0
%define release 1
%if 0%{?python3_version_nodots} < 36
%define __python3 %{__python3_other}
%define python3_version %{python3_other_version}
%define python3_version_nodots %{python3_other_version_nodots}
%define py3_build %{py3_other_build}
%define py3_install %{py3_other_install}
%define python3_sitelib %{python3_other_sitelib}
%endif
Name: %{name}
Version: %{version}
Release: %{release}%{?dist}
Summary: Python library for interacting with the Koji RPM build system
Group: Development/Libraries
License: GPL-3.0-or-later
Url: https://git.ligo.org/adam-mercer/koji-packager/
Source0: %{name}-%{version}.tar.gz
Packager: Duncan Macleod <duncan.macleod@ligo.org>
BuildArch: noarch
# build dependencies
BuildRequires: rpm-build
BuildRequires: python-rpm-macros
BuildRequires: python3-rpm-macros
BuildRequires: python%{python3_version_nodots}-setuptools
BuildRequires: help2man
# testing dependencies (python3x only)
BuildRequires: python%{python3_version_nodots}-pytest >= 2.8.0
%description
FIXME
# -- python3x-gwkoji
%package -n python%{python3_version_nodots}-%{name}
Summary: Python %{python3_version} library for interacting with the Koji RPM build system
%{?python_provide:%python_provide python%{python3_version_nodots}-%{name}}
%description -n python%{python3_version_nodots}-%{name}
FIXME
# -- build steps
%prep
%autosetup -n %{name}-%{version}
%build
%py3_build
%install
%py3_install
# make man page for gwkoji-packager
mkdir -vp %{buildroot}%{_mandir}/man1
env PYTHONPATH="%{buildroot}%{python3_sitelib}" \
help2man \
--source %{name} \
--version-string %{version} \
--section 1 --no-info --no-discard-stderr \
--output %{buildroot}%{_mandir}/man1/gwkoji-packager.1 \
%{buildroot}%{_bindir}/gwkoji-packager
%check
%{__python3} -m pytest --pyargs %{name}
%{__python3} -m gwkoji.packager --help
PYTHONPATH="%{buildroot}%{python3_sitelib}" %{buildroot}%{_bindir}/gwkoji-packager --help
%files -n python%{python3_version_nodots}-%{name}
%doc README.rst
%license COPYING
%{python3_sitelib}/*
%{_bindir}/gwkoji-packager
%{_mandir}/man1/gwkoji-packager.1*
# -- changelog
%changelog
* Tue May 14 2019 Duncan Macleod <duncan.macleod@ligo.org> - 0.1.0-1
- first cut at packaging gwkoji
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Adam Mercer <adam.mercer@ligo.org>
#
# This file is part of gwkoji.
#
# gwkoji is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Python library for interacting with the Koji RPM build system
"""
__version__ = "0.1.0"
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Adam Mercer <adam.mercer@ligo.org>
#
# This file is part of gwkoji.
#
# gwkoji is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os.path
import pathlib
import re
import shutil
import tempfile
from distutils.spawn import find_executable
from .utils import (
logged_check_call,
logged_check_output,
source_type,
)
RPMBUILD_TS_RESP = re.compile(r'\AWrote: (?P<srcrpm>\S+)\Z')
def _rpmbuild_args(*args, tmpdir=None):
"""Construct a command-line call to ``rpmbuild``
"""
rpmbargs = [find_executable('rpmbuild'), '--verbose']
if tmpdir:
rpmbargs.extend(('--define', '_topdir {0}'.format(tmpdir)))
return rpmbargs + list(args)
def download_spec_sources(spec, outdir, logger=None):
"""Run spectool to download sources defined in a spec file
Parameters
----------
spec : `str`, `pathlib.Path`
path to spec file
outdir : `str`, `pathlib.Path`
output directory for downloaded files
"""
spectool = find_executable('spectool')
cmd = [
spectool,
"--get-files",
"--directory",
str(outdir),
str(spec),
]
return logged_check_call(
cmd,
logger=logger,
)
def build_src_rpm(source, *extra_rpmbuild_args, outdir=os.path.curdir,
logger=None):
"""Create a src RPM from a tarball or a spec file
Parameters
----------
source : `str`, `pathlib.Path`
path to source file, either a tarball or a spec file
*extra_rpmbuild_args
positional arguments are passed as command-line arguments to
``rpmbuild``
outdir : `str`, `pathlib.Path`, optional
output directory for ``src.rpm`` file
Returns
-------
srcrpm : `str`
the path of the new `src.rpm` file
"""
type_ = source_type(source)
with tempfile.TemporaryDirectory() as tmpdir:
tmppath = pathlib.Path(tmpdir)
sourcepath = tmppath / "SOURCES"
sourcepath.mkdir()
# run spectool to download sources
if type_ == "spec":
download_spec_sources(source, sourcepath, logger=logger)
# run rpmbuild
rpmcmd = "-bs" if type_ == "spec" else "-ts"
cmd = _rpmbuild_args(*extra_rpmbuild_args + (rpmcmd, str(source)),
tmpdir=tmpdir)
resp = logged_check_output(cmd, logger=logger).strip().decode('utf-8')
# copy src.rpm into outdir and return path
srcrpm = RPMBUILD_TS_RESP.match(resp).groupdict()['srcrpm']
return shutil.move(srcrpm, outdir)
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Adam Mercer <adam.mercer@ligo.org>
#
# This file is part of gwkoji.
#
# gwkoji is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Interactions with koji
"""
from distutils.spawn import find_executable
from .utils import logged_check_call
def _koji_call(
*args,
**kwargs,
):
koji = find_executable("koji")
if koji is None:
raise FileNotFoundError("No such file or directory: 'koji'")
cmd = (koji,) + args
return logged_check_call(cmd, **kwargs)
def moshimoshi(*args, **kwargs):
return _koji_call("moshimoshi", *args, **kwargs)
def list_pkgs(*args, **kwargs):
return _koji_call("list-pkgs", *args, **kwargs)
def add_pkg(*args, **kwargs):
return _koji_call("add-pkg", *args, **kwargs)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Adam Mercer <adam.mercer@ligo.org>
#
# This file is part of Koji-Packager.
# This file is part of gwkoji.
#
# Koji-Packager is free software: you can redistribute it and/or modify
# gwkoji is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
......@@ -23,44 +22,67 @@
import argparse
import atexit
import os.path
import logging
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
from distutils.spawn import find_executable
from . import (
__version__,
koji,
)
from .build import build_src_rpm
from .utils import source_type
LOGGER_FORMAT = "[%(asctime)-15s] %(name)s | %(levelname)+8s | %(message)s"
LOGGER_DATEFMT = "%Y-%m-%d %H:%M:%S"
logging.basicConfig(format=LOGGER_FORMAT, datefmt=LOGGER_DATEFMT)
logger = logging.getLogger("gwkoji")
try:
import coloredlogs
except ImportError:
set_logger_level = logger.setLevel
else:
coloredlogs.install(
logger=logger,
fmt=LOGGER_FORMAT,
datefmt=LOGGER_DATEFMT,
)
set_logger_level = coloredlogs.set_level
#
# metadata
#
__version__ = "0.0.1"
__target__ = "sandbox"
DEFAULT_TARGET = "sandox"
#
# helper methods
#
RPMBUILD_TS_RESP = re.compile(r'\AWrote: (?P<srcrpm>\S+)\Z')
def parse_options():
# define option parser
parser = argparse.ArgumentParser(description='Build RPMs using Koji')
parser.add_argument('-v', '--verbose', action='store_true',
help='increase verbosity level')
parser.add_argument('-V', '--version', action='version', version=__version__)
parser.add_argument('-V', '--version', action='version',
version=__version__,
help='show version number and exit')
parser.add_argument('-v', '--verbose', action='count', default=2,
help='show verbose output, give once for INFO, '
'give twice for DEBUG')
# koji options
koji_options = parser.add_argument_group('Koji Options')
koji_options.add_argument('--target', default=__target__,
help='koji tag to tag (%s)' % __target__)
koji_options.add_argument('--target', default=DEFAULT_TARGET,
help='koji tag to tag (default: %(default)s)')
koji_options.add_argument('--owner', help='koji package owner')
koji_options.add_argument('--scratch', action='store_true',
help='perform a scratch build')
koji_options.add_argument('--scratch', action='store_true', default=False,
help='perform a scratch build '
'(default: %(default)s)')
# source options
source_options = parser.add_argument_group('Source Options')
......@@ -105,132 +127,6 @@ def parse_options():
return args
def source_type(source):
"""Determine the source type based on its file name
Parameters
----------
source : `pathlib.Path`
the source `Path`
Returns
-------
type_ : `str`
the source type, one of
- ``'git'``
- ``'spec'``
- ``'srcrpm'``
- ``'tarball'``
Raises
------
ValueError
if the source type is not recognised
"""
name = source.name
if name.endswith(".src.rpm"):
return "srcrpm"
if name.endswith(".spec"):
return "spec"
if name.endswith(".tar") or source.stem.endswith(".tar"):
return "tarball"
if name.endswith(".git"):
return "git"
raise ValueError(
"failed to determine source type for '{0}'".format(source),
)
def _rpmbuild_args(*args, tmpdir=None):
"""Construct a command-line call to ``rpmbuild``
"""
rpmbargs = [find_executable('rpmbuild')]
if tmpdir:
rpmbargs.extend(('--define', '_topdir {0}'.format(tmpdir)))
return rpmbargs + list(args)
def download_spec_sources(spec, outdir):
"""Run spectool to download sources defined in a spec file
Parameters
----------
spec : `str`, `pathlib.Path`
path to spec file
outdir : `str`, `pathlib.Path`
output directory for downloaded files
"""
spectool = find_executable('spectool')
cmd = [
spectool,
"--get-files",
"--directory",
str(outdir),
str(spec),
]
return subprocess.check_call(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def build_src_rpm(source, *extra_rpmbuild_args, outdir=os.path.curdir):
"""Create a src RPM from a tarball or a spec file
Parameters
----------
source : `str`, `pathlib.Path`
path to source file, either a tarball or a spec file
*extra_rpmbuild_args
positional arguments are passed as command-line arguments to
``rpmbuild``
outdir : `str`, `pathlib.Path`, optional
output directory for ``src.rpm`` file
Returns
-------
srcrpm : `str`
the path of the new `src.rpm` file
"""
type_ = source_type(source)
with tempfile.TemporaryDirectory() as tmpdir:
tmppath = pathlib.Path(tmpdir)
sourcepath = tmppath / "SOURCES"
sourcepath.mkdir()
# run spectool to download sources
if type_ == "spec":
download_spec_sources(source, sourcepath)
# run rpmbuild
rpmcmd = "-bs" if type_ == "spec" else "-ts"
cmd = _rpmbuild_args(*extra_rpmbuild_args + (rpmcmd, source),
tmpdir=tmpdir)
resp = subprocess.check_output(cmd).strip().decode('utf-8')
# copy src.rpm into outdir and return path
srcrpm = RPMBUILD_TS_RESP.match(resp).groupdict()['srcrpm']
return shutil.move(srcrpm, outdir)
# method to check for connection to koji
def check_connection_to_koji():
try:
cmd = ['koji', 'moshimoshi']
subprocess.check_call(cmd, stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError: