Skip to content
Snippets Groups Projects
Commit 19b9ec80 authored by Robert Bruntz's avatar Robert Bruntz
Browse files

Merge branch 'cicd-components' into 'master' 

Update CI/CD pipeline to use components

See merge request !126
parents 4f3a0595 b456c995
No related branches found
No related tags found
1 merge request!126Update CI/CD pipeline to use components
Pipeline #701273 passed with warnings
Stage: build
Stage: test
Stage: .post
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 78
− 21
View file @ 19b9ec80
stages:
- Source distribution
- Source packages
- Binary packages
- Testing
- Documentation
- Code coverage
- Code quality
include: include:
# tarball # -- Python ------------------------
- local: '/.gitlab/ci/dist.yml'
# RHEL packaging # Build tarball (sdist) and wheel, and run tests with PyTest for various
- local: '/.gitlab/ci/rhel.yml' # Python versions, with coverage and test reports
# Debian packaging # https://computing.docs.ligo.org/guide/gitlab/components/python/all/
- local: '/.gitlab/ci/debian.yml' - component: git.ligo.org/computing/gitlab/components/python/all@2
# Python tests inputs:
- local: '/.gitlab/ci/python.yml' # quality assurance
# Code coverage code_quality_analyzer: flake8
- local: '/.gitlab/ci/coverage.yml' run_advanced_sast: true
# Code quality # build-and-test
- local: '/.gitlab/ci/analysis.yml' python_versions:
- "3.9"
- "3.10"
- "3.11"
pytest_options: >-
--cov=dqsegdb
-ra
--verbose
extra_test_commands: &extra_test_commands
# sanity check the scripts
- PREFIX=$(python3 -c "import sys; print(sys.prefix)")
- for entrypoint in $(ls ${PREFIX}/bin/*dqsegdb);
do
echo -e "\x1B[92m$ ${entrypoint} --help...\x1B[0m";
python3 -m coverage run
--append
--source=dqsegdb
${entrypoint} --help
; done
# -- Debian packaging --------------
# Build Debian source and binary packages from the tarball (separately)
# and runs test jobs
# https://computing.docs.ligo.org/guide/gitlab/components/debian/all/
- component: git.ligo.org/computing/gitlab/components/debian/all@2
inputs:
needs: [sdist]
debian_versions:
- bullseye
- bookworm
test_script: &system_test_script
# run pytest using the installed test suite (dqsegdb.tests)
- python3 -m pytest
--cov=dqsegdb
--junit-xml=junit.xml
--pyargs
-ra
--verbose
dqsegdb.tests
# run the same 'extra' commands to check the scripts
- *extra_test_commands
test_install: &system_test_install >-
dqsegdb
python3-pytest
python3-pytest-cov
# -- Red Hat packaging -------------
# Build RPMs from the tarball and runs test jobs
# https://computing.docs.ligo.org/guide/gitlab/components/redhat/all/
- component: git.ligo.org/computing/gitlab/components/redhat/all@2
inputs:
needs: [sdist]
redhat_versions:
- 8
test_script: *system_test_script
test_install: *system_test_install
# -- customisations ------------------
# allow Debian test jobs to fail until this project can be updated to
# support lscsoft-glue >= 4.0.0.
debian_test:
allow_failure: true
.gitlab/ci/analysis.yml deleted 100644 → 0
+ 0
− 28
View file @ 4f3a0595
# ---------------------------
# Code Quality workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
file:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
- python.yml
# -- code quality -----------
#
# These jobs check the code for
# quality issues
#
flake8:
stage: Code quality
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:flake8
- .python:flake8
needs: []
variables:
# don't fail the pipeline because of linting issues,
# these are presented in the code-quality box in the
# merge_request UI
FLAKE8_OPTIONS: "--exit-zero"
.gitlab/ci/coverage.yml deleted 100644 → 0
+ 0
− 22
View file @ 4f3a0595
# ---------------------------
# Code coverage
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# -- coverage ---------------
#
# This job combinces the individual
# coverage reports from each job
# to make a proper coverage result
#
coverage:
stage: Code coverage
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:coverage-combine
- .python:coverage-combine
.gitlab/ci/debian.yml deleted 100644 → 0
+ 0
− 163
View file @ 4f3a0595
# ---------------------------
# Debian packaging workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/
file: debian.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- macros
.buster:
image: igwn/builder:buster
.bullseye:
image: igwn/builder:bullseye
# -- source packages --------
#
# These jobs make DSC packages
#
.dsc:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:dsc
- .debian:dsc
stage: Source packages
needs:
- tarball
variables:
TARBALL: "dqsegdb-*.tar.*"
before_script:
- !reference [".debian:dsc", "before_script"]
# install dch
- apt-get -yqq install devscripts
# -- hack the tarball so that setuptools-scm version matches
# the debian changelog
- tar -zxf ${TARBALL}
# get version from the Python metadata
- PKG_VERSION=$(grep ^Version dqsegdb-*/PKG-INFO | cut -d\ -f2)
# add a changelog entry for this version
- (cd dqsegdb-*/; dch --force-bad-version --newversion ${PKG_VERSION}-9999 --controlmaint "ci build")
# repack the tarball
- tar -zcf ${TARBALL} */
dsc:buster:
extends:
- .dsc
- .buster
dsc:bullseye:
extends:
- .dsc
- .bullseye
# -- binary packages --------
#
# These jobs generate DEB
# binary packages from the
# DSC sources packages
#
.deb:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:deb
- .debian:deb
stage: Binary packages
variables:
DSC: "dqsegdb_*.dsc"
deb:buster:
extends:
- .deb
- .buster
needs:
- dsc:buster
deb:bullseye:
extends:
- .deb
- .bullseye
needs:
- dsc:bullseye
# -- test -------------------
.test:debian:
extends:
# see /.gitlab/ci/test.yml
- .test
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:base
- .debian:base
before_script:
# set up apt
- !reference [".debian:base", before_script]
# setup local apt repository
- apt-get -y -q -q install local-apt-repository
- mkdir -pv /srv/local-apt-repository
# fill our local apt repo and rebuild it
- mv -v *.deb /srv/local-apt-repository
- /usr/lib/local-apt-repository/rebuild
- apt-get -y -q update
# install our package(s)
- apt-get -y install python3-dqsegdb dqsegdb
# install testing dependencies
- apt-get -y -q install
findutils
python3-coverage
python3-pip
python3-pytest
python3-pytest-cov
python3-requests-mock
# upgrade coverage.py to >=5 so that .coverage files use new format
- python3 -m pip install "coverage>=5" --upgrade-strategy=only-if-needed
test:buster:
extends:
- .test:debian
- .buster
image: igwn/base:buster
needs:
- deb:buster
test:bullseye:
extends:
- .test:debian
- .bullseye
image: igwn/base:bullseye
needs:
- deb:bullseye
# -- lint -------------------
#
# These jobs check the code
# for quality issues
#
.lintian:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/debian/#.debian:lint
- .debian:lint
stage: Code quality
variables:
LINTIAN_OPTIONS: "--color always --suppress-tags new-package-should-close-itp-bug,groff-message --allow-root --pedantic"
lintian:buster:
extends:
- .lintian
- .buster
needs:
- deb:buster
variables:
LINTIAN_OPTIONS: "--color always --suppress-tags new-package-should-close-itp-bug,latest-debian-changelog-entry-without-new-version --allow-root --pedantic"
lintian:bullseye:
extends:
- .lintian
- .bullseye
needs:
- deb:bullseye
.gitlab/ci/dist.yml deleted 100644 → 0
+ 0
− 26
View file @ 4f3a0595
# ---------------------------
# Distribution workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# -- dist -------------------
#
# This job makes the dqsegdb-X.Y.Z.tar.gz
# distribution and uploads it as a job
# artifact
#
tarball:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:build
- .python:build
image: python:3
stage: Source distribution
variables:
SDIST: "true"
WHEEL: "false"
.gitlab/ci/python.yml deleted 100644 → 0
+ 0
− 66
View file @ 4f3a0595
# ---------------------------
# Python workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/python/
file: python.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- build ------------------
#
# this job builds the wheel
# distribution
#
wheel:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:build
- .python:build
image: python:3
stage: Binary packages
variables:
SDIST: "false"
WHEEL: "true"
# -- test -------------------
#
# These jobs run the tests
#
.test:pip:
extends:
# see /.gitlab/ci/test.yml
- .test
needs:
- wheel
variables:
INSTALL_TARGET: "dqsegdb-*.whl"
test:python3.6:
extends:
- .test:pip
image: python:3.6
test:python3.7:
extends:
- .test:pip
image: python:3.7
test:python3.8:
extends:
- .test:pip
image: python:3.8
test:python3.9:
extends:
- .test:pip
image: python:3.9
test:python3.10:
extends:
- .test:pip
image: python:3.10
.gitlab/ci/rhel.yml deleted 100644 → 0
+ 0
− 159
View file @ 4f3a0595
# ---------------------------
# RHEL packaging workflow
# ---------------------------
include:
# https://computing.docs.ligo.org/gitlab-ci-templates/
- project: computing/gitlab-ci-templates
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/
file: rhel.yml
# local test template
- local: /.gitlab/ci/test.yml
# -- macros
.el7:
image: igwn/builder:el7-testing
variables:
EPEL: "true"
.el8:
image: igwn/builder:el8-testing
variables:
EPEL: "true"
# -- source packages --------
#
# These jobs make src RPMs
#
.srpm:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:srpm
- .rhel:srpm
stage: Source packages
needs:
- tarball
variables:
TARBALL: "dqsegdb-*.tar.*"
before_script:
- !reference [".rhel:srpm", "before_script"]
# hack the spec file to use the correct package version
- tar --file ${TARBALL} --wildcards --strip-components 1 --get dqsegdb*/etc/*.spec dqsegdb*/PKG-INFO
- PKG_VERSION=$(grep ^Version PKG-INFO | cut -d\ -f2)
- sed -i 's|define version\( *\)\(.*\)|define unmangled_version '${PKG_VERSION}'\n%define version '${PKG_VERSION/-/+}'|' etc/*.spec
- sed -i 's|pypi_source|pypi_source %{srcname} %{unmangled_version}|' etc/*.spec
script:
- rpmbuild -bs
--define "_srcrpmdir ${CI_PROJECT_DIR}"
--define "_sourcedir ${CI_PROJECT_DIR}"
etc/*.spec
srpm:el7:
extends:
- .srpm
- .el7
srpm:el8:
extends:
- .srpm
- .el8
# -- binary packages --------
#
# These jobs generate binary RPMs
# from the src RPMs
#
.rpm:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:rpm
- .rhel:rpm
stage: Binary packages
variables:
SRPM: "python-dqsegdb-*.src.rpm"
rpm:el7:
extends:
- .rpm
- .el7
needs:
- srpm:el7
rpm:el8:
extends:
- .rpm
- .el8
needs:
- srpm:el8
# -- test -------------------
.test:el:
extends:
# see /.gitlab/ci/test.yml
- .test
before_script:
# set up yum caching
- !reference [".rhel:base", before_script]
# configure EPEL
- yum -y -q install epel-release && yum -y -q install epel-rpm-macros
# install testing dependencies
- PY3=$(rpm --eval '%{?python3_pkgversion:%{python3_pkgversion}}%{!?python3_pkgversion:3}')
- yum -y -q install
findutils
python${PY3}-pip
python${PY3}-pytest
python${PY3}-pytest-cov
# install our package(s)
- yum -y -q install *.rpm
# upgrade coverage.py to >=5 so that .coverage files use new format
- python3 -m pip install "coverage>=5" --upgrade-strategy=only-if-needed
artifacts:
reports:
junit: null
test:el7:
extends:
- .test:el
- .el7
image: igwn/base:el7-testing
needs:
- rpm:el7
test:el8:
extends:
- .test:el
- .el8
image: igwn/base:el8-testing
needs:
- rpm:el8
# -- lint -------------------
#
# These jobs check the code
# for quality issues
#
.rpmlint:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/rhel/#.rhel:lint
- .rhel:lint
stage: Code quality
variables:
GIT_STRATEGY: fetch
RPMLINT_OPTIONS: '--info --file .rpmlintrc'
rpmlint:el7:
extends:
- .rpmlint
- .el7
needs:
- rpm:el7
rpmlint:el8:
extends:
- .rpmlint
- .el8
needs:
- rpm:el8
.gitlab/ci/test.yml deleted 100644 → 0
+ 0
− 42
View file @ 4f3a0595
# ---------------------------
# Testing workflow
# ---------------------------
# test template used in other jobs
# see python.yml for example
.test:
extends:
# https://computing.docs.ligo.org/gitlab-ci-templates/python/#.python:pytest
- .python:pytest
stage: Testing
variables:
# target for coverage
COVERAGE_TARGET: "dqsegdb"
# don't need the git repo
GIT_STRATEGY: none
# configure pytest
PYTEST_OPTIONS: "-ra -v --pyargs dqsegdb.tests"
# use python3
PYTHON: "python3"
script:
# run the tests
- !reference [".python:pytest", script]
# run --help on each one (with coverage)
- PREFIX=$(${PYTHON} -c "import sys; print(sys.prefix)")
- for entrypoint in $(ls ${PREFIX}/bin/*dqsegdb);
do
echo -e "\x1B[92m$ ${entrypoint} --help...\x1B[0m";
${PYTHON} -m coverage run
--append
--source=dqsegdb
${entrypoint} --help
; done
# upload coverage file as an artifact to be combined later
artifacts:
when: always
paths:
- .coverage-*
# disable coverage total for test jobs, this is reported
# by the coverage job
coverage: null
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment