.gitlab-ci.yml 34.3 KB
Newer Older
1
2
3

# -- setup --------------------------------------

Adam Mercer's avatar
Adam Mercer committed
4
5
image: docker:latest

6
7
8
9
10
11
12
13
14
15
stages:
  - LAL
  - LALFrame
  - LALMetaIO
  - LALSimulation
  - LALBurst
  - LALInspiral
  - LALPulsar
  - LALInference
  - LALApps
Duncan Macleod's avatar
Duncan Macleod committed
16
  - lint
17
18
19
20
  - documentation
  - integration tests
  - compiler tests
  - platform tests
Duncan Macleod's avatar
Duncan Macleod committed
21
  - upgrade tests
22
23
24
25
  - docker
  - wheels
  - deploy

Adam Mercer's avatar
Adam Mercer committed
26
variables:
27
  APT_GET: "apt-get -y -q -q -o dir::cache::archives=${CI_PROJECT_DIR}/.cache/apt"
Adam Mercer's avatar
Adam Mercer committed
28
  BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
29
  COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
30
31
  CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
  CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
Duncan Macleod's avatar
Duncan Macleod committed
32
  CPU_COUNT: 4
33
34
35
36
37
38
39
  DOCKER_DRIVER: overlay
  GIT_DEPTH: 1
  LAL_DIR: $CI_PROJECT_DIR/opt/lalsuite
  NIGHTLY: $CI_REGISTRY_IMAGE:nightly
  TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
  YUM: "yum -y -q --setopt=cachedir=${CI_PROJECT_DIR}/.cache/yum --setopt=keepcache=1"
  YUM_OPTS: "-y -q --setopt=cachedir=${CI_PROJECT_DIR}/.cache/yum --setopt=keepcache=1"
Adam Mercer's avatar
Adam Mercer committed
40
41

before_script:
42
  - set -x  # print commands as they are run (helps debugging)
Adam Mercer's avatar
Adam Mercer committed
43
44
  - ulimit -S -c 0
  - export VERBOSE="true"
Adam Mercer's avatar
Adam Mercer committed
45
  - export PATH=/usr/lib/ccache:/opt/local/libexec/ccache:$PATH
Adam Mercer's avatar
Adam Mercer committed
46
47
48
  - export CCACHE_DIR=${PWD}/ccache
  - export PKG_CONFIG_PATH=${LAL_DIR}/lib/pkgconfig
  - mkdir -p opt/lalsuite
49
  - if [ ${CI_PIPELINE_SOURCE} = "schedule" ] || [ ${CI_PIPELINE_SOURCE} = "web" ]; then EXTRA_CONFIG_FLAGS="--enable-nightly"; fi
Adam Mercer's avatar
Adam Mercer committed
50
51
52
53
54
55

cache:
  key: $CI_JOB_NAME
  paths:
    - ccache

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
.retry: &define-retry |
  retry() {
    local n=1
    local max=3
    local delay=15
    while true; do
      "$@" && break || {
        if [[ $n -lt $max ]]; then
          ((n++))
          echo "Command failed. Attempt $n/$max:"
          sleep $delay;
        else
          echo "The command has failed after $n attempts." 1>&2
          exit 1
        fi
      }
    done
  }
Adam Mercer's avatar
Adam Mercer committed
74

75
76
77
78
.top-level-build: &top-level-build
  script:
    - ./00boot
    - ./configure --enable-doxygen --enable-swig
Duncan Macleod's avatar
Duncan Macleod committed
79
    - make -j${CPU_COUNT} VERBOSE=1 distcheck
Duncan Macleod's avatar
Duncan Macleod committed
80
81
  artifacts:
    reports:
82
83
      junit:
        - "*/_build/*/test/python/junit*.xml"
84

85
# -- build templates ----------------------------
86

87
# -- rpm
88

89
90
91
92
93
94
.yum-cache: &yum-cache
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/yum

95
.yum-update: &yum-update
96
  yum-config-manager --disable htcondor 1>/dev/null && ${YUM} update
97

98
.rpmbuild: &rpmbuild
99
  <<: *yum-cache
100
  image: igwn/base:el7-testing
Adam Mercer's avatar
Adam Mercer committed
101
  variables:
102
    GIT_STRATEGY: none
103
    RPM_BUILD_NCPUS: ${CPU_COUNT}
Adam Mercer's avatar
Adam Mercer committed
104
  script:
105
    - PACKAGE=${CI_JOB_NAME%%:*}
106
    # install srpm dependencies
107
    - *yum-update
108
    - ${YUM} install lscsoft-packaging-tools
109
    # install upstream packages
110
    - if [ -d rpmbuild ]; then ${YUM} install rpmbuild/RPMS/x86_64/*.rpm; fi
111
    # build src.rpm
112
113
    - cd ${PACKAGE}/
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
114
    - rpmbuild -ts --define "_topdir $CI_PROJECT_DIR/rpmbuild" ${TARBALL}
115
    - SRCRPM=${CI_PROJECT_DIR}/rpmbuild/SRPMS/${PACKAGE}-*.src.rpm
116
    # install build dependencies
117
    - yum-builddep ${YUM_OPTS} ${SRCRPM}
118
119
    # build binary rpms and print details of what we got
    - rpmbuild --rebuild --define "_topdir $CI_PROJECT_DIR/rpmbuild" ${SRCRPM}
120
121
122
123
124
125
126
127
128
129
130
131
    # print package info
    - set +x
    - for rpmf in ${CI_PROJECT_DIR}/rpmbuild/RPMS/*/*${PACKAGE}-*.rpm; do
          echo "===== ${rpmf}" &&
          rpm -qlp "${rpmf}" &&
          echo "Files:" &&
          rpm -qip "${rpmf}" &&
          echo "Provides:" &&
          rpm -qp --provides "${rpmf}" &&
          echo "Requires:" &&
          rpm -qp --requires "${rpmf}";
      done
Duncan Macleod's avatar
Duncan Macleod committed
132
    # lint RPMs
Duncan Macleod's avatar
Duncan Macleod committed
133
134
135
136
137
138
139
140
141
142
    - |
      cat << EOF > rpmlintrc
      # don't validate Source0
      setOption("NetworkEnabled", False)
      # don't know how to fix this
      addFilter('binary-or-shlib-defines-rpath')
      # the regex rpmlint uses to identify 'lib' libraries is crap
      addFilter('explicit-lib-dependency (.*)?matplotlib')
      addFilter('explicit-lib-dependency (.*)?ciecplib')
      EOF
Duncan Macleod's avatar
Duncan Macleod committed
143
    - rpmlint
Duncan Macleod's avatar
Duncan Macleod committed
144
          -f rpmlintrc
Duncan Macleod's avatar
Duncan Macleod committed
145
          ${CI_PROJECT_DIR}/rpmbuild/RPMS/*/*${PACKAGE}-*.rpm
Adam Mercer's avatar
Adam Mercer committed
146
  artifacts:
Adam Mercer's avatar
Adam Mercer committed
147
    expire_in: 18h
Adam Mercer's avatar
Adam Mercer committed
148
    paths:
149
      - "rpmbuild/BUILD/*/config.log"
150
151
152
      - "rpmbuild/BUILD/*/test-suite.log"
      - "rpmbuild/BUILD/*/test/test-suite.log"
      - "rpmbuild/BUILD/*/test/python/test-suite.log"
153
      - "rpmbuild/RPMS/x86_64/${CI_JOB_NAME%%:*}-*.rpm"
154
      - "rpmbuild/RPMS/x86_64/lib${CI_JOB_NAME%%:*}-*.rpm"
155
      - "rpmbuild/RPMS/x86_64/python*-${CI_JOB_NAME%%:*}-*.rpm"
Duncan Macleod's avatar
Duncan Macleod committed
156
    reports:
157
      junit: "rpmbuild/BUILD/*/test/python/junit*.xml"
158
    when: always
Adam Mercer's avatar
Adam Mercer committed
159
  only:
160
161
162
163
164
165
166
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip rhel\]/
167

168
# -- debian
169

170
171
172
173
174
175
176
177
178
179
180
.apt-cache: &apt-cache
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/apt

.apt-update: &apt-update
  mkdir -p "${CI_PROJECT_DIR}/.cache/apt/partial" &&
  ${APT_GET} autoclean &&
  ${APT_GET} update

181
.debuild: &debbuild
182
  <<: *apt-cache
183
  image: igwn/base:buster
184
  variables:
Duncan Macleod's avatar
Duncan Macleod committed
185
    DEB_BUILD_OPTIONS: "parallel=${CPU_COUNT}"
186
    GIT_STRATEGY: none
187
  script:
188
    - PACKAGE=${CI_JOB_NAME%%:*}
189
    # install debs from artifacts
190
191
192
193
194
195
    - *apt-update
    - ${APT_GET} install
          devscripts
          dpkg-dev
          lintian
    - "ls *.deb &>/dev/null && { dpkg -i *.deb || { ${APT_GET} -f install; dpkg -i *.deb; } }"
196
    # create orig tarball
197
198
    - cd ${PACKAGE}/
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
199
200
    - SUFFIX=$(basename $TARBALL | sed 's/.*\.\(tar\..*\)/\1/')
    - VERSION=$(basename $TARBALL | sed 's/[^-]*-\(.*\)\.tar\..*/\1/' | tr '-' '~')
201
    - cd ${CI_PROJECT_DIR}/
202
    - cp ${TARBALL} ${PACKAGE}_${VERSION}.orig.${SUFFIX}
203
204
205
    # update changelog
    - export DEBFULLNAME="GitLab"
    - export DEBEMAIL="gitlab@git.ligo.org"
206
    - tar -xf ${TARBALL}
207
    - cd ${PACKAGE}-*/
208
    - dch -v ${VERSION}-1 -b 'Rebuilt automatically on git.ligo.org CI'
209
    # install build dependencies
210
211
212
213
    - mk-build-deps
          --tool "${APT_GET} -o Debug::pkgProblemResolver=yes --no-install-recommends"
          --install
          --remove
214
    # build packages
Duncan Macleod's avatar
Duncan Macleod committed
215
216
    - debuild
         -us -uc -r
217
         --lintian-opts --color=always --allow-root
218
219
220
221
222
223
224
225
    # print package info
    - set +x
    - cd ${CI_PROJECT_DIR}
    - for debf in *.deb; do
          echo "===== ${debf}";
          dpkg --info "${debf}";
          dpkg --contents "${debf}";
      done
226
  artifacts:
Adam Mercer's avatar
Adam Mercer committed
227
    expire_in: 18h
228
    paths:
229
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/config.log
230
231
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/test-suite.log
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/python/test-suite.log
232
233
      - ${CI_JOB_NAME%%:*}*.changes
      - ${CI_JOB_NAME%%:*}*.deb
234
      - lib${CI_JOB_NAME%%:*}*.deb
235
236
237
      - python*-${CI_JOB_NAME%%:*}*.deb
      - ${CI_JOB_NAME%%:*}*.dsc
      - ${CI_JOB_NAME%%:*}*.orig.*
Duncan Macleod's avatar
Duncan Macleod committed
238
    reports:
239
      junit: "${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/python/junit*.xml"
240
    when: always
241
  only:
242
243
244
245
246
247
248
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip debian\]/
249

250
# -- conda
251

252
253
.conda-init: &conda-init |
  # configure conda
Duncan Macleod's avatar
Duncan Macleod committed
254
  mkdir -p $(dirname ${CONDA_PKGS_DIRS})
255
256
257
  source /opt/conda/etc/profile.d/conda.sh
  conda config --set always_yes yes
  conda config --add channels conda-forge
258
  retry conda update -n base conda setuptools
259
  conda activate base
260
  retry conda install -n base "conda-build!=3.18.10" conda-forge-pinning "conda-smithy>=3.7.5" conda-verify
261
262
  conda info --all

263
.conda-build: &conda-build
264
  image: igwn/base:conda
265
266
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
267
    CONDA_CONFIG: "linux_64_"
268
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
269
    GIT_STRATEGY: none
270
  before_script:
271
    - *define-retry
272
    - *conda-init
273
    - conda config --set channel_priority strict
274
  script:
275
    - PACKAGE=${CI_JOB_NAME%%:*}
276
    - cd ${PACKAGE}/
277
    # render clobber YAML file to use our tarball
278
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
279
    - SHA256=$(openssl dgst -r -sha256 $TARBALL | cut -d\  -f1)
280
281
    - tar -xf ${TARBALL} --wildcards ${PACKAGE}-*/conda/ --strip-components=1
    - sed 's|@TARBALL@|'${TARBALL}'|g' conda/meta.yaml.in > conda/meta.yaml
282
    - sed -i 's|@SHA256@|'${SHA256}'|g' conda/meta.yaml
283
284
285
286
287
    # create a feedstock from the conda recipe
    - git config --global user.name "${GITLAB_USER_NAME}"
    - git config --global user.email "${GITLAB_USER_EMAIL}"
    - conda smithy init conda/ --feedstock-directory ${PACKAGE}-feedstock
    - cd ${PACKAGE}-feedstock
288
    - retry conda smithy regenerate --no-check-uptodate
289
    - git ls-files
290
    # build packages (preferring local packages over everything)
291
292
293
    # note: we use a variable and `eval` to get around the
    #       _pyversion string having spaces in it
    - cmd="conda build
294
          recipe/
Duncan Macleod's avatar
Duncan Macleod committed
295
          --dirty
296
297
          --error-overlinking
          --error-overdepending
Duncan Macleod's avatar
Duncan Macleod committed
298
          --keep-old-work
299
          --no-anaconda-upload
300
          --use-channeldata
301
          --use-local
302
          --variant-config-files .ci_support/${CONDA_CONFIG}.yaml
303
304
305
306
307
308
309
310
          ${CONDA_BUILD_ARGS}"
    # if not a nightly or web-triggered pipeline, just build one python version
    - if [ ${CI_PIPELINE_SOURCE} != "schedule" ] && [ ${CI_PIPELINE_SOURCE} != "web" ]; then
         _pyversion=$(python -c "import yaml; conf = yaml.load(open('.ci_support/${CONDA_CONFIG}.yaml'), Loader=yaml.SafeLoader); print(conf['python'][-1])");
         cmd="${cmd} --python \"${_pyversion}\"";
      fi
    # run the build
    - eval $cmd
311
312
313
314
  after_script:
    # clean cache of old files
    - find ${CONDA_PKGS_DIRS%:*} -atime +30 -delete
    - find ${CONDA_PKGS_DIRS%:*} -type d -empty -delete
315
316
317
  artifacts:
    expire_in: 18h
    paths:
318
319
      - "conda/bld/linux-64/${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/${CI_JOB_NAME%%:*}-*.tar.bz2"
320
321
      - "conda/bld/linux-64/lib${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/lib${CI_JOB_NAME%%:*}-*.tar.bz2"
322
323
      - "conda/bld/linux-64/python-${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/python-${CI_JOB_NAME%%:*}-*.tar.bz2"
324
325
326
327
328
329
      - "conda/bld/broken/${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/${CI_JOB_NAME%%:*}-*.tar.bz2"
      - "conda/bld/broken/lib${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/lib${CI_JOB_NAME%%:*}-*.tar.bz2"
      - "conda/bld/broken/python-${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/python-${CI_JOB_NAME%%:*}-*.tar.bz2"
330
331
332
333
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/config.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/test-suite.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/python/test-suite.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/*/*/*/*/test-suite.log"
334
      - "${CI_JOB_NAME%%:*}/${CI_JOB_NAME%%:*}-feedstock/"
Duncan Macleod's avatar
Duncan Macleod committed
335
    reports:
Duncan Macleod's avatar
Duncan Macleod committed
336
      junit:
337
338
        - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/python/junit*.xml"
        - "conda/bld/${CI_JOB_NAME%%:*}-*/test_tmp/junit*.xml"
339
    when: always
340
  cache:
341
    key: $CI_JOB_NAME
342
343
    paths:
      - conda/pkgs
344
345
346
347
348
349
350
351
  only:
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip conda\]/
352

Duncan Macleod's avatar
Duncan Macleod committed
353
354
# -- linting ------------------------------------

355
.lint:
Duncan Macleod's avatar
Duncan Macleod committed
356
  stage: lint
357
  needs: []
358
359
360
361
362
363
364
365
  only:
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip lint\]/

lint:python:
  extends:
    - .lint
  image: python
Duncan Macleod's avatar
Duncan Macleod committed
366
367
368
369
370
371
372
373
374
375
376
377
378
379
  before_script:
    - python -m pip install
          "flake8>=3.7.0"
          "flake8-junit-report"
  script:
    - python -m flake8 --output-file flake8.out
  after_script:
    - cat flake8.out
    - python -m junit_conversor flake8.out junit.xml
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/pip
  artifacts:
380
    expire_in: 1month
Duncan Macleod's avatar
Duncan Macleod committed
381
382
383
384
    reports:
      junit: junit.xml
    when: always

Adam Mercer's avatar
Adam Mercer committed
385
lint:coala:
386
387
  extends:
    - .lint
Adam Mercer's avatar
Adam Mercer committed
388
  image: coala/base
389
390
  before_script:
    - pip install coala-json
Adam Mercer's avatar
Adam Mercer committed
391
  script:
Adam Mercer's avatar
Adam Mercer committed
392
393
394
    # run once to show output to console
    - coala --ci || true
    # run again to prepare JSON report for upload
Adam Mercer's avatar
Adam Mercer committed
395
    - coala --ci --json -o output.json
396
397
398
399
400
401
402
403
404
405
406
  after_script:
    - coala-json --junit -f output.json -o junit.xml
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/pip
  artifacts:
    expire_in: 1month
    reports:
      junit: junit.xml
    when: always
Adam Mercer's avatar
Adam Mercer committed
407

408
409
410
# -- documentation ------------------------------

documentation:
411
  image: igwn/lalsuite-dev:buster
412
  stage: documentation
Adam Mercer's avatar
Adam Mercer committed
413
  needs: []
414
415
416
  script:
    - ./00boot
    - ./configure --enable-doxygen --prefix=${LAL_DIR}
Duncan Macleod's avatar
Duncan Macleod committed
417
    - make -j${CPU_COUNT} install-html
418
419
420
421
422
423
424
425
426
    - pushd ${LAL_DIR}/share/doc
    - cp lalsuite/index.html index.html
    - sed -i 's/..\/lal/lal/g' index.html
    - popd
  artifacts:
    expire_in: 18h
    paths:
      - ${LAL_DIR}/share/doc
  only:
427
428
429
430
431
    refs:
      - pushes
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip docs\]/
432
433

pages:
434
  image: igwn/lalsuite-dev:buster
435
436
437
438
  stage: deploy
  script:
    - ./00boot
    - ./configure --enable-doxygen --prefix=${LAL_DIR}
Duncan Macleod's avatar
Duncan Macleod committed
439
    - make -j${CPU_COUNT} install-html
440
441
442
443
444
    - cp -r ${LAL_DIR}/share/doc public/
    - pushd public
    - cp lalsuite/index.html index.html
    - sed -i 's/..\/lal/lal/g' index.html
    - popd
445
446
447
    - cp coverage.html public/
  needs:
    - coverage
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
  artifacts:
    paths:
      - public
  only:
    - master@lscsoft/lalsuite
  except:
    - pushes
    - web

# -- tarballs -----------------------------------

# this job makes all of the source tarballs up-front using a top-level
# make dist call
#
# to use one (any) of the tarballs in another job, just add "make-dist"
# in `dependencies:`

make-dist:
466
  image: igwn/lalsuite-dev:el7
467
468
469
470
471
472
473
474
475
476
477
478
479
  stage: .pre
  script:
    - ./00boot
    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
    - for subdir in lal lalframe lalmetaio lalsimulation lalburst lalinspiral lalpulsar lalinference lalapps; do
        pushd ${subdir};
        make dist;
        popd;
      done
  artifacts:
    expire_in: 18h
    paths:
      - "*/*.tar.*"
480

481
482
483
484
485
486
487
488
489
490
# just make the lal tarball to release other jobs
make-dist:lal:
  extends:
    - make-dist
  script:
    - pushd lal
    - ./00boot
    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
    - make dist

491
492
493
494
495
# -- lal ----------------------------------------

lal:rpm:
  <<: *rpmbuild
  stage: LAL
496
  needs:
497
    - make-dist:lal
498

499
500
501
lal:deb:
  <<: *debbuild
  stage: LAL
502
  needs:
503
    - make-dist:lal
504

505
lal:conda:fftw:
506
507
  <<: *conda-build
  stage: LAL
508
  needs:
509
    - make-dist:lal
510
511
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
512
    CONDA_CONFIG: "linux_64_fft_implfftw"
513
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
514

515
516
517
518
lal:conda:mkl:
  <<: *conda-build
  stage: LAL
  needs:
519
    - make-dist:lal
520
521
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
522
    CONDA_CONFIG: "linux_64_fft_implmkl"
523
524
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"

525
526
527
528
529
# -- lalframe------------------------------------

lalframe:rpm:
  <<: *rpmbuild
  stage: LALFrame
530
  needs:
531
    - make-dist
532
    - lal:rpm
533

534
535
536
lalframe:deb:
  <<: *debbuild
  stage: LALFrame
537
  needs:
538
    - make-dist
539
    - lal:deb
540

541
542
543
lalframe:conda:
  <<: *conda-build
  stage: LALFrame
544
  needs:
545
    - make-dist
546
    - lal:conda:fftw
547

548
549
550
551
552
# -- lalmetaio ----------------------------------

lalmetaio:rpm:
  <<: *rpmbuild
  stage: LALMetaIO
553
  needs:
554
    - make-dist
555
    - lal:rpm
556

557
558
559
lalmetaio:deb:
  <<: *debbuild
  stage: LALMetaIO
560
  needs:
Duncan Macleod's avatar
Duncan Macleod committed
561
    - make-dist
562
    - lal:deb
Duncan Macleod's avatar
Duncan Macleod committed
563

564
565
566
567
568
lalmetaio:conda:
  <<: *conda-build
  stage: LALMetaIO
  needs:
    - make-dist
569
    - lal:conda:fftw
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591

# -- lalsimulation ------------------------------

lalsimulation:rpm:
  <<: *rpmbuild
  stage: LALSimulation
  needs:
    - make-dist
    - lal:rpm

lalsimulation:deb:
  <<: *debbuild
  stage: LALSimulation
  needs:
    - make-dist
    - lal:deb

lalsimulation:conda:
  <<: *conda-build
  stage: LALSimulation
  needs:
    - make-dist
592
    - lal:conda:fftw
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618

# -- lalburst -----------------------------------

lalburst:rpm:
  <<: *rpmbuild
  stage: LALBurst
  needs:
    - make-dist
    - lal:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm

lalburst:deb:
  <<: *debbuild
  stage: LALBurst
  needs:
    - make-dist
    - lal:deb
    - lalmetaio:deb
    - lalsimulation:deb

lalburst:conda:
  <<: *conda-build
  stage: LALBurst
  needs:
    - make-dist
619
    - lal:conda:fftw
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
    - lalmetaio:conda
    - lalsimulation:conda

# -- lalinspiral --------------------------------

lalinspiral:rpm:
  <<: *rpmbuild
  stage: LALInspiral
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm

lalinspiral:deb:
  <<: *debbuild
  stage: LALInspiral
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb

lalinspiral:conda:
  <<: *conda-build
  stage: LALInspiral
  needs:
    - make-dist
652
    - lal:conda:fftw
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda

# -- lalpulsar ----------------------------------

lalpulsar:rpm:
  <<: *rpmbuild
  stage: LALPulsar
  needs:
    - make-dist
    - lal:rpm

lalpulsar:deb:
  <<: *debbuild
  stage: LALPulsar
  needs:
    - make-dist
    - lal:deb

lalpulsar:conda:
  <<: *conda-build
  stage: LALPulsar
  needs:
    - make-dist
679
    - lal:conda:fftw
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713

# -- lalinference -------------------------------

lalinference:rpm:
  <<: *rpmbuild
  stage: LALInference
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm

lalinference:deb:
  <<: *debbuild
  stage: LALInference
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb

lalinference:conda:
  <<: *conda-build
  stage: LALInference
  needs:
    - make-dist
714
    - lal:conda:fftw
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda
    - lalpulsar:conda
    - lalinspiral:conda

# -- lalapps ------------------------------------

lalapps:rpm:
  <<: *rpmbuild
  stage: LALApps
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm

lalapps:deb:
  <<: *debbuild
  stage: LALApps
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb

lalapps:conda:
  <<: *conda-build
  stage: LALApps
  needs:
    - make-dist
757
    - lal:conda:fftw
758
759
760
761
762
763
764
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda
    - lalpulsar:conda
    - lalinspiral:conda
    - lalinference:conda
765
766
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
767
    CONDA_CONFIG: "linux_64_python3.7.____cpython"
768
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
769
770
771

# -- integration tests --------------------------

772
773
774
775
776
777
.integration-tests: &integration-tests
  stage: integration tests
  only:
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip integration\]/

778
lalinference_testjob:
779
  <<: *yum-cache
780
781
  extends:
    - .integration-tests
782
  image: igwn/base:el7-testing
783
784
  before_script:
    - *yum-update
785
786
787
    - ${YUM} install
          git
          git-lfs
788
    - ${YUM} install rpmbuild/RPMS/x86_64/*.rpm
789
790
791
792
793
794
795
796
797
798
799
800
801
  script:
    - git lfs clone https://git.ligo.org/lscsoft/ROQ_data --include "**/params.dat,*/4s/**"
    - bash lalinference/test/lalinference_nestedSampling_integration_test.sh
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
  only:
802
803
804
    refs:
      - schedules
      - web
805
806
807
808
809
  artifacts:
    expire_in: 18h
    paths:
      - example/
    when: always
810

Adam Mercer's avatar
Adam Mercer committed
811
top-level:
812
  image: igwn/lalsuite-dev:el7-testing
Adam Mercer's avatar
Adam Mercer committed
813
  needs: []
814
  <<: *top-level-build
815
816
  extends:
    - .integration-tests
817
  only:
818
819
820
821
    refs:
      - pushes
      - schedules
      - web
822

823
824
# Note: adapted from https://git.ligo.org/lscsoft/ligo.skymap.
coverage:
825
826
  extends:
    - .integration-tests
827
  image: igwn/lalsuite-dev-gcc:8
828
829
830
831
832
833
834
835
  needs: []
  script:
    - apt-get -y install --no-install-recommends python3-pip
    - python3 -m pip install coverage gcovr pycobertura
    - ./00boot
    # Configure for C code coverage measurement with gcc
    - ./configure --enable-swig-python CFLAGS="--coverage -fprofile-abs-path" PYTHON=python3
    # Run unit tests with Python scripts instrumented by coverage.py
Duncan Macleod's avatar
Duncan Macleod committed
836
    - make -j${CPU_COUNT} check COVERAGE_FILE="$(pwd)/.coverage" PY_LOG_FLAGS="-m coverage run --source \"$(pwd)\" --omit \*/test/\* -p"
837
    # Output C coverage data in Cobertura format
Duncan Macleod's avatar
Duncan Macleod committed
838
    - gcovr -j${CPU_COUNT} --exclude-directories test --xml c-coverage.xml
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
    # Combine and output Python coverage data in Cobertura format
    - coverage combine
    - coverage xml -o py-coverage.xml
    # Merge coverage reports. They're just XML, after all.
    - |
      python3 - <<EOF
      import lxml.etree
      import copy

      doc1 = lxml.etree.parse('py-coverage.xml')
      doc2 = lxml.etree.parse('c-coverage.xml')
      root1 = doc1.getroot()
      root2 = doc2.getroot()
      root1.attrib['lines-covered'] = str(
          int(root1.attrib['lines-covered']) +
          int(root2.attrib['lines-covered']))
      root1.attrib['lines-valid'] = str(
          int(root1.attrib['lines-valid']) +
          int(root2.attrib['lines-valid']))
      try:
          root1.attrib['line-rate'] = str(
              int(root1.attrib['lines-covered']) /
              int(root1.attrib['lines-valid']))
      except ZeroDivisionError:
          root1.attrib['line-rate'] = '0'
      root1.attrib['branches-covered'] = str(
          int(root1.attrib['branches-covered']) +
          int(root2.attrib['branches-covered']))
      root1.attrib['branches-valid'] = str(
          int(root1.attrib['branches-valid']) +
          int(root2.attrib['branches-valid']))
      try:
          root1.attrib['branch-rate'] = str(
              int(root1.attrib['branches-covered']) /
              int(root1.attrib['branches-valid']))
      except ZeroDivisionError:
          root1.attrib['branch-rate'] = '0'
      packages = root1.find('./packages')
      packages.extend(root2.iterfind('./packages/package'))
      doc1.write('coverage.xml')
      EOF
    # Write HTML coverage report and show coverage summary.
    - pycobertura show coverage.xml -f html -o coverage.html
    - pycobertura show coverage.xml | tail -n 1
  coverage: '/TOTAL\s+.*\s+([\d\.]+)%/'
  artifacts:
    paths:
      - coverage.html
    reports:
      cobertura: coverage.xml
  only:
890
891
892
893
    refs:
      - pushes
      - schedules
      - web
894

895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
# -- docker -------------------------------------

# build a nightly container from the RPMs
docker:nightly:el7:
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add RPMs to directory to pass to docker
    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
    - rm -rf rpmbuild*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker build --pull -t $CI_REGISTRY_IMAGE/nightly:el7 --file .gitlab-ci-el7.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/nightly:el7
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
    - lalapps:rpm
  only:
    - schedules
922

923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
# build a tagged container from the rpms
docker:tags:el7:
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add RPMs to directory to pass to docker
    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
    - rm -rf rpmbuild*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker build --pull -t $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:el7 --file .gitlab-ci-el7.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:el7
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
    - lalapps:rpm
  only:
    - /^lalsuite-v.*$/

# build a nightly container from the debian packages
950
docker:nightly:buster:
951
952
953
954
955
956
957
958
959
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add deb packages to directory to pass to docker
    - mkdir debs && mv *.deb debs
    - rm *.changes *.dsc *.orig.*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
960
961
    - docker build --pull -t $CI_REGISTRY_IMAGE/nightly:buster --file .gitlab-ci-buster.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/nightly:buster
962
963
964
965
966
967
968
969
970
971
972
973
974
975
  needs:
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb
    - lalapps:deb
  only:
    - schedules

# build a tagged container from the debian packages
976
docker:tags:buster:
977
978
979
980
981
982
983
984
985
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add deb packages to directory to pass to docker
    - mkdir debs && mv *.deb debs
    - rm *.changes *.dsc *.orig.*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
986
987
    - docker build --pull -t $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:buster --file .gitlab-ci-buster.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:buster
988
989
990
991
992
993
994
995
996
997
998
999
1000
  needs:
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb
    - lalapps:deb
  only:
    - /^lalsuite-v.*$/

For faster browsing, not all history is shown. View entire blame