.gitlab-ci.yml 7.64 KB
Newer Older
Leo P. Singer's avatar
Leo P. Singer committed
1
stages:
2
3
  - dist
  - test
Leo P. Singer's avatar
Leo P. Singer committed
4
  - deploy
Leo P. Singer's avatar
Leo P. Singer committed
5

6
7
8
9
#
# Build Python source package.
#

Leo P. Singer's avatar
Leo P. Singer committed
10
sdist:
11
12
  image: python
  stage: dist
Leo P. Singer's avatar
Leo P. Singer committed
13
  script:
14
    - python setup.py sdist -d .
Leo P. Singer's avatar
Leo P. Singer committed
15
16
  artifacts:
    paths:
17
      - '*.tar.*'
Leo P. Singer's avatar
Leo P. Singer committed
18
    expire_in: 3 hours
Leo P. Singer's avatar
Leo P. Singer committed
19

20
#
Leo P. Singer's avatar
Leo P. Singer committed
21
# Build binary wheels for Linux and macOS.
22
23
#

Leo P. Singer's avatar
Leo P. Singer committed
24
25
26
27
28
.wheel:manylinux1: &wheel-manylinux1
  # This container is derived from the official manylinux image provided by
  # python.org (see PEP 513), and includes all of the LALSuite
  # build-dependencies.
  image: containers.ligo.org/lscsoft/lalsuite-manylinux:master
29
  stage: dist
Leo P. Singer's avatar
Leo P. Singer committed
30
31
32
33
  script:
    # Build and install LALSuite
    - PYPREFIX=/opt/python/$(echo ${CI_JOB_NAME} | sed 's/.*:\(.*\)-manylinux1/\1/')
    - ${PYPREFIX}/bin/python setup.py bdist_wheel
Leo P. Singer's avatar
Leo P. Singer committed
34
    - auditwheel repair dist/*.whl
35
    - rm dist/*
36
    - mv wheelhouse/* .
Leo P. Singer's avatar
Leo P. Singer committed
37
38
  artifacts:
    paths:
39
      - '*.whl'
Leo P. Singer's avatar
Leo P. Singer committed
40
    expire_in: 3 hours
Leo P. Singer's avatar
Leo P. Singer committed
41

Leo P. Singer's avatar
Leo P. Singer committed
42
43
.wheel:macos: &wheel-macos
  tags:
44
    - macos_elcapitan
45
  stage: dist
Leo P. Singer's avatar
Leo P. Singer committed
46
47
48
49
50
  script:
    - PYVERS=$(echo ${CI_JOB_NAME} | sed 's/.*:cp\(.\)\(.\).*/\1.\2/')
    # Enter virtualenv so that we have a controlled version of Numpy
    - virtualenv-${PYVERS} env
    - source env/bin/activate
51
    # FIXME: https://github.com/matthew-brett/delocate/pull/38
52
    - pip install -q git+https://github.com/lpsinger/delocate@namespace-packages#egg=delocate 'numpy==1.14.5;python_version>="3.7"' 'numpy==1.7.0;python_version<"3.7"'
Leo P. Singer's avatar
Leo P. Singer committed
53
54
55
    # Build and audit wheel
    - python setup.py bdist_wheel
    - delocate-wheel -v -w wheelhouse dist/*.whl
56
57
    - rm -f *.whl
    - mv wheelhouse/* .
Leo P. Singer's avatar
Leo P. Singer committed
58
59
  artifacts:
    paths:
60
      - '*.whl'
Leo P. Singer's avatar
Leo P. Singer committed
61
    expire_in: 3 hours
Leo P. Singer's avatar
Leo P. Singer committed
62

Leo P. Singer's avatar
Leo P. Singer committed
63
64
wheel:cp35-cp35m-manylinux1:
  <<: *wheel-manylinux1
Leo P. Singer's avatar
Leo P. Singer committed
65
66
wheel:cp36-cp36m-manylinux1:
  <<: *wheel-manylinux1
67
68
wheel:cp37-cp37m-manylinux1:
  <<: *wheel-manylinux1
Leo P. Singer's avatar
Leo P. Singer committed
69
70
wheel:cp35-cp35m-macosx:
  <<: *wheel-macos
71
72
wheel:cp36-cp36m-macosx:
  <<: *wheel-macos
73
74
wheel:cp37-cp37m-macosx:
  <<: *wheel-macos
Leo P. Singer's avatar
Leo P. Singer committed
75

76
77
78
79
80
81
82
#
# Build Docker containers for dependencies listed in requirements.txt,
# plus dependencies for running the unit tests, collecting coverage data,
# and generating the docs.
#

.dependencies: &dependencies
83
  stage: dist
84
85
86
  variables:
    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
  script:
87
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
88
89
90
91
92
    - |
      cat <<EOF > Dockerfile
      FROM python:${CI_JOB_NAME#*python}
      COPY requirements.txt .
      RUN apt-get update -qq && apt-get -y install --no-install-recommends libchealpix-dev libgsl0-dev pkg-config && rm -rf /var/lib/apt/lists/*
93
      RUN pip --no-cache-dir install pytest-astropy pytest-xdist pytest-cov gcovr pycobertura sphinx sphinx-argparse sphinx-astropy flake8
94
95
      RUN pip --no-cache-dir install -r requirements.txt
      RUN rm -f requirements.txt
Leo P. Singer's avatar
Leo P. Singer committed
96
      EOF
97
98
    - docker build -t $IMAGE_TAG .
    - docker push $IMAGE_TAG
99

Leo P. Singer's avatar
Leo P. Singer committed
100
101
102
dependencies/python3.5:
  <<: *dependencies

103
104
105
dependencies/python3.6:
  <<: *dependencies

106
107
108
dependencies/python3.7:
  <<: *dependencies

109
110
111
112
113
#
# Generate documentation.
#

docs:
114
  image: $CI_REGISTRY_IMAGE/dependencies/python3.7:$CI_COMMIT_REF_NAME
115
116
117
118
119
120
121
122
123
124
125
  stage: test
  variables:
    GIT_STRATEGY: none
  script:
    - tar --strip-components 1 -xf *.tar.*
    - python setup.py build_docs
  dependencies:
    - sdist
  artifacts:
    paths:
      - docs/_build/html/
Leo P. Singer's avatar
Leo P. Singer committed
126
    expire_in: 3 hours
127
128
129
130
131
132
133
134
135
136

#
# Test the wheels.
#

.test: &test
  variables:
    GIT_STRATEGY: none
  script:
    - pip install *.whl
137
    - python -c 'import sys; from ligo.skymap import test; sys.exit(test(verbose=True))'
138

Leo P. Singer's avatar
Leo P. Singer committed
139
140
141
142
143
144
test/python3.5:
  <<: *test
  image: $CI_REGISTRY_IMAGE/dependencies/python3.5:$CI_COMMIT_REF_NAME
  dependencies:
    - wheel:cp35-cp35m-manylinux1

145
146
147
148
149
150
test/python3.6:
  <<: *test
  image: $CI_REGISTRY_IMAGE/dependencies/python3.6:$CI_COMMIT_REF_NAME
  dependencies:
    - wheel:cp36-cp36m-manylinux1

151
152
test/python3.7:
  <<: *test
153
  image: $CI_REGISTRY_IMAGE/dependencies/python3.7:$CI_COMMIT_REF_NAME
154
155
156
  dependencies:
    - wheel:cp37-cp37m-manylinux1

157
158
#
# Measure test coverage:
159
160
161
162
163
164
165
166
167
168
# - coverage.py for Python code
# - gcov/gcovr for C code
#
# Export the results from both to Cobertura format because it's an XML format
# that both coverage.py and gcovr can write, merge them by hand, and then
# write HTML and text summaries.
#
# This would be a lot prettier if we could use coveralls or codecov.io,
# which support multilingual test coverage. However, those products don't
# integrate with git.ligo.org (or at least, they don't integrate for free).
169
170
171
172
#

test/coverage:
  stage: test
173
  image: $CI_REGISTRY_IMAGE/dependencies/python3.7:$CI_COMMIT_REF_NAME
174
175
176
  variables:
    CFLAGS: -coverage
    GIT_STRATEGY: none
177
  coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
178
  before_script:
179
    - tar --strip-components 1 -xf *.tar.*
180
  script:
181
    # Run tests.
182
    - python setup.py test --coverage -V
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
    # Write coverage reports in Cobertura format.
    - gcovr build/temp*/src -r . -x -o c-coverage.xml
    - coverage xml -o py-coverage.xml
    # Merge coverage reports. They're just XML, after all.
    - |
      python - <<EOF
      import lxml.etree
      import copy

      doc1 = lxml.etree.parse('py-coverage.xml')
      doc2 = lxml.etree.parse('c-coverage.xml')
      root1 = doc1.getroot()
      root2 = doc2.getroot()
      root1.attrib['lines-covered'] = str(
          int(root1.attrib['lines-covered']) +
          int(root2.attrib['lines-covered']))
      root1.attrib['lines-valid'] = str(
          int(root1.attrib['lines-valid']) +
          int(root2.attrib['lines-valid']))
      try:
          root1.attrib['line-rate'] = str(
              int(root1.attrib['lines-covered']) /
              int(root1.attrib['lines-valid']))
      except ZeroDivisionError:
          root1.attrib['line-rate'] = '0'
      root1.attrib['branches-covered'] = str(
          int(root1.attrib['branches-covered']) +
          int(root2.attrib['branches-covered']))
      root1.attrib['branches-valid'] = str(
          int(root1.attrib['branches-valid']) +
          int(root2.attrib['branches-valid']))
      try:
          root1.attrib['branch-rate'] = str(
              int(root1.attrib['branches-covered']) /
              int(root1.attrib['branches-valid']))
      except ZeroDivisionError:
          root1.attrib['branch-rate'] = '0'
      packages = root1.find('./packages')
      packages.extend(root2.iterfind('./packages/package'))
      doc1.write('coverage.xml')
      EOF
    # Write human-readable report.
    - pycobertura show coverage.xml -f html -o coverage.html
    - pycobertura show coverage.xml
227
228
  dependencies:
    - sdist
229
230
  artifacts:
    paths:
231
232
      - coverage.xml
      - coverage.html
Leo P. Singer's avatar
Leo P. Singer committed
233
    expire_in: 3 hours
Leo P. Singer's avatar
Leo P. Singer committed
234

235
236
237
238
239
#
# Run flake8 linter to enforce code style.
#

lint:
240
  image: $CI_REGISTRY_IMAGE/dependencies/python3.7:$CI_COMMIT_REF_NAME
241
  stage: test
Leo P. Singer's avatar
Leo P. Singer committed
242
243
244
  variables:
    GIT_STRATEGY: none
  script:
245
246
247
248
249
250
251
252
    - tar --strip-components 1 -xf *.tar.*
    - flake8 --show-source .
  dependencies:
    - sdist

#
# Gather coverage reports and docs for GitLab pages and build badges.
#
253

254
pages:
Leo P. Singer's avatar
Leo P. Singer committed
255
256
  variables:
    GIT_STRATEGY: none
257
258
  stage: deploy
  script:
Leo P. Singer's avatar
Leo P. Singer committed
259
    - mv docs/_build/html public
260
    - mv coverage.html public/coverage.html
Leo P. Singer's avatar
Leo P. Singer committed
261
  dependencies:
Leo P. Singer's avatar
Leo P. Singer committed
262
    - docs
263
    - test/coverage
264
265
266
267
268
269
  artifacts:
    paths:
      - public
    expire_in: 30 days
  only:
    - master
270
271
272
273
274
275
276
277
278
279
280
281

#
# Upload to PyPI.
#

deploy:wheel:
  stage: deploy
  image: containers.ligo.org/lscsoft/lalsuite-manylinux:master
  variables:
    GIT_STRATEGY: none
  script:
    # TWINE_USERNAME and TWINE_PASSWORD are provided by CI secret variables
282
283
    - /opt/python/cp37-cp37m/bin/pip install twine
    - /opt/python/cp37-cp37m/bin/twine upload *
284
285
  dependencies:
    - sdist
Leo P. Singer's avatar
Leo P. Singer committed
286
    - wheel:cp35-cp35m-manylinux1
287
    - wheel:cp36-cp36m-manylinux1
288
    - wheel:cp37-cp37m-manylinux1
Leo P. Singer's avatar
Leo P. Singer committed
289
    - wheel:cp35-cp35m-macosx
290
    - wheel:cp36-cp36m-macosx
291
    - wheel:cp37-cp37m-macosx
292
293
  only:
    - tags