.gitlab-ci.yml 34.2 KB
Newer Older
1
2
3

# -- setup --------------------------------------

Adam Mercer's avatar
Adam Mercer committed
4
5
image: docker:latest

6
7
8
9
10
11
12
13
14
15
stages:
  - LAL
  - LALFrame
  - LALMetaIO
  - LALSimulation
  - LALBurst
  - LALInspiral
  - LALPulsar
  - LALInference
  - LALApps
Duncan Macleod's avatar
Duncan Macleod committed
16
  - lint
17
18
19
20
  - documentation
  - integration tests
  - compiler tests
  - platform tests
Duncan Macleod's avatar
Duncan Macleod committed
21
  - upgrade tests
22
23
24
25
  - docker
  - wheels
  - deploy

Adam Mercer's avatar
Adam Mercer committed
26
variables:
27
  APT_GET: "apt-get -y -q -q -o dir::cache::archives=${CI_PROJECT_DIR}/.cache/apt"
Adam Mercer's avatar
Adam Mercer committed
28
  BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
29
  COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
30
31
  CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
  CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
Duncan Macleod's avatar
Duncan Macleod committed
32
  CPU_COUNT: 4
33
34
35
36
37
38
39
  DOCKER_DRIVER: overlay
  GIT_DEPTH: 1
  LAL_DIR: $CI_PROJECT_DIR/opt/lalsuite
  NIGHTLY: $CI_REGISTRY_IMAGE:nightly
  TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
  YUM: "yum -y -q --setopt=cachedir=${CI_PROJECT_DIR}/.cache/yum --setopt=keepcache=1"
  YUM_OPTS: "-y -q --setopt=cachedir=${CI_PROJECT_DIR}/.cache/yum --setopt=keepcache=1"
Adam Mercer's avatar
Adam Mercer committed
40
41

before_script:
42
  - set -x  # print commands as they are run (helps debugging)
Adam Mercer's avatar
Adam Mercer committed
43
44
  - ulimit -S -c 0
  - export VERBOSE="true"
Adam Mercer's avatar
Adam Mercer committed
45
  - export PATH=/usr/lib/ccache:/opt/local/libexec/ccache:$PATH
Adam Mercer's avatar
Adam Mercer committed
46
47
48
  - export CCACHE_DIR=${PWD}/ccache
  - export PKG_CONFIG_PATH=${LAL_DIR}/lib/pkgconfig
  - mkdir -p opt/lalsuite
49
  - if [ ${CI_PIPELINE_SOURCE} = "schedule" ] || [ ${CI_PIPELINE_SOURCE} = "web" ]; then EXTRA_CONFIG_FLAGS="--enable-nightly"; fi
Adam Mercer's avatar
Adam Mercer committed
50
51
52
53
54
55

cache:
  key: $CI_JOB_NAME
  paths:
    - ccache

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
.retry: &define-retry |
  retry() {
    local n=1
    local max=3
    local delay=15
    while true; do
      "$@" && break || {
        if [[ $n -lt $max ]]; then
          ((n++))
          echo "Command failed. Attempt $n/$max:"
          sleep $delay;
        else
          echo "The command has failed after $n attempts." 1>&2
          exit 1
        fi
      }
    done
  }
Adam Mercer's avatar
Adam Mercer committed
74

75
76
77
78
.top-level-build: &top-level-build
  script:
    - ./00boot
    - ./configure --enable-doxygen --enable-swig
Duncan Macleod's avatar
Duncan Macleod committed
79
    - make -j${CPU_COUNT} VERBOSE=1 distcheck
Duncan Macleod's avatar
Duncan Macleod committed
80
81
  artifacts:
    reports:
82
83
      junit:
        - "*/_build/*/test/python/junit*.xml"
84

85
# -- build templates ----------------------------
86

87
# -- rpm
88

89
90
91
92
93
94
.yum-cache: &yum-cache
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/yum

95
.yum-update: &yum-update
96
  yum-config-manager --disable htcondor 1>/dev/null && ${YUM} update
97

98
.rpmbuild: &rpmbuild
99
  <<: *yum-cache
100
  image: igwn/base:el7-testing
Adam Mercer's avatar
Adam Mercer committed
101
  variables:
102
    GIT_STRATEGY: none
103
    RPM_BUILD_NCPUS: ${CPU_COUNT}
Adam Mercer's avatar
Adam Mercer committed
104
  script:
105
    - PACKAGE=${CI_JOB_NAME%%:*}
106
    # install srpm dependencies
107
    - *yum-update
108
    - ${YUM} install lscsoft-packaging-tools
109
    # install upstream packages
110
    - if [ -d rpmbuild ]; then ${YUM} install rpmbuild/RPMS/x86_64/*.rpm; fi
111
    # build src.rpm
112
113
    - cd ${PACKAGE}/
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
114
    - rpmbuild -ts --define "_topdir $CI_PROJECT_DIR/rpmbuild" ${TARBALL}
115
    - SRCRPM=${CI_PROJECT_DIR}/rpmbuild/SRPMS/${PACKAGE}-*.src.rpm
116
    # install build dependencies
117
    - yum-builddep ${YUM_OPTS} ${SRCRPM}
118
119
    # build binary rpms and print details of what we got
    - rpmbuild --rebuild --define "_topdir $CI_PROJECT_DIR/rpmbuild" ${SRCRPM}
120
121
122
123
124
125
126
127
128
129
130
131
    # print package info
    - set +x
    - for rpmf in ${CI_PROJECT_DIR}/rpmbuild/RPMS/*/*${PACKAGE}-*.rpm; do
          echo "===== ${rpmf}" &&
          rpm -qlp "${rpmf}" &&
          echo "Files:" &&
          rpm -qip "${rpmf}" &&
          echo "Provides:" &&
          rpm -qp --provides "${rpmf}" &&
          echo "Requires:" &&
          rpm -qp --requires "${rpmf}";
      done
Duncan Macleod's avatar
Duncan Macleod committed
132
    # lint RPMs
Duncan Macleod's avatar
Duncan Macleod committed
133
134
135
136
137
138
139
140
141
142
    - |
      cat << EOF > rpmlintrc
      # don't validate Source0
      setOption("NetworkEnabled", False)
      # don't know how to fix this
      addFilter('binary-or-shlib-defines-rpath')
      # the regex rpmlint uses to identify 'lib' libraries is crap
      addFilter('explicit-lib-dependency (.*)?matplotlib')
      addFilter('explicit-lib-dependency (.*)?ciecplib')
      EOF
Duncan Macleod's avatar
Duncan Macleod committed
143
    - rpmlint
Duncan Macleod's avatar
Duncan Macleod committed
144
          -f rpmlintrc
Duncan Macleod's avatar
Duncan Macleod committed
145
          ${CI_PROJECT_DIR}/rpmbuild/RPMS/*/*${PACKAGE}-*.rpm
Adam Mercer's avatar
Adam Mercer committed
146
  artifacts:
Adam Mercer's avatar
Adam Mercer committed
147
    expire_in: 18h
Adam Mercer's avatar
Adam Mercer committed
148
    paths:
149
      - "rpmbuild/BUILD/*/config.log"
150
151
152
      - "rpmbuild/BUILD/*/test-suite.log"
      - "rpmbuild/BUILD/*/test/test-suite.log"
      - "rpmbuild/BUILD/*/test/python/test-suite.log"
153
      - "rpmbuild/RPMS/x86_64/${CI_JOB_NAME%%:*}-*.rpm"
154
      - "rpmbuild/RPMS/x86_64/lib${CI_JOB_NAME%%:*}-*.rpm"
155
      - "rpmbuild/RPMS/x86_64/python*-${CI_JOB_NAME%%:*}-*.rpm"
Duncan Macleod's avatar
Duncan Macleod committed
156
    reports:
157
      junit: "rpmbuild/BUILD/*/test/python/junit*.xml"
158
    when: always
Adam Mercer's avatar
Adam Mercer committed
159
  only:
160
161
162
163
164
165
166
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip rhel\]/
167

168
# -- debian
169

170
171
172
173
174
175
176
177
178
179
180
.apt-cache: &apt-cache
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/apt

.apt-update: &apt-update
  mkdir -p "${CI_PROJECT_DIR}/.cache/apt/partial" &&
  ${APT_GET} autoclean &&
  ${APT_GET} update

181
.debuild: &debbuild
182
  <<: *apt-cache
183
  image: igwn/base:buster
184
  variables:
Duncan Macleod's avatar
Duncan Macleod committed
185
    DEB_BUILD_OPTIONS: "parallel=${CPU_COUNT}"
186
    GIT_STRATEGY: none
187
  script:
188
    - PACKAGE=${CI_JOB_NAME%%:*}
189
    # install debs from artifacts
190
191
192
193
194
195
    - *apt-update
    - ${APT_GET} install
          devscripts
          dpkg-dev
          lintian
    - "ls *.deb &>/dev/null && { dpkg -i *.deb || { ${APT_GET} -f install; dpkg -i *.deb; } }"
196
    # create orig tarball
197
198
    - cd ${PACKAGE}/
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
199
200
    - SUFFIX=$(basename $TARBALL | sed 's/.*\.\(tar\..*\)/\1/')
    - VERSION=$(basename $TARBALL | sed 's/[^-]*-\(.*\)\.tar\..*/\1/' | tr '-' '~')
201
    - cd ${CI_PROJECT_DIR}/
202
    - cp ${TARBALL} ${PACKAGE}_${VERSION}.orig.${SUFFIX}
203
204
205
    # update changelog
    - export DEBFULLNAME="GitLab"
    - export DEBEMAIL="gitlab@git.ligo.org"
206
    - tar -xf ${TARBALL}
207
    - cd ${PACKAGE}-*/
208
    - dch -v ${VERSION}-1 -b 'Rebuilt automatically on git.ligo.org CI'
209
    # install build dependencies
210
211
212
213
    - mk-build-deps
          --tool "${APT_GET} -o Debug::pkgProblemResolver=yes --no-install-recommends"
          --install
          --remove
214
    # build packages
Duncan Macleod's avatar
Duncan Macleod committed
215
216
    - debuild
         -us -uc -r
217
         --lintian-opts --color=always --allow-root
218
219
220
221
222
223
224
225
    # print package info
    - set +x
    - cd ${CI_PROJECT_DIR}
    - for debf in *.deb; do
          echo "===== ${debf}";
          dpkg --info "${debf}";
          dpkg --contents "${debf}";
      done
226
  artifacts:
Adam Mercer's avatar
Adam Mercer committed
227
    expire_in: 18h
228
    paths:
229
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/config.log
230
231
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/test-suite.log
      - ${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/python/test-suite.log
232
233
      - ${CI_JOB_NAME%%:*}*.changes
      - ${CI_JOB_NAME%%:*}*.deb
234
      - lib${CI_JOB_NAME%%:*}*.deb
235
236
237
      - python*-${CI_JOB_NAME%%:*}*.deb
      - ${CI_JOB_NAME%%:*}*.dsc
      - ${CI_JOB_NAME%%:*}*.orig.*
Duncan Macleod's avatar
Duncan Macleod committed
238
    reports:
239
      junit: "${CI_JOB_NAME%%:*}*/.pybuild/python*/build/test/python/junit*.xml"
240
    when: always
241
  only:
242
243
244
245
246
247
248
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip debian\]/
249

250
# -- conda
251

252
253
.conda-init: &conda-init |
  # configure conda
Duncan Macleod's avatar
Duncan Macleod committed
254
  mkdir -p $(dirname ${CONDA_PKGS_DIRS})
255
256
257
  source /opt/conda/etc/profile.d/conda.sh
  conda config --set always_yes yes
  conda config --add channels conda-forge
258
  retry conda update -n base conda setuptools
259
  conda activate base
260
  retry conda install -n base "conda-build!=3.18.10" conda-forge-pinning "conda-smithy>=3.7.5" conda-verify
261
262
  conda info --all

263
.conda-build: &conda-build
264
  image: igwn/base:conda
265
266
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
267
    CONDA_CONFIG: "linux_64_"
268
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
269
    GIT_STRATEGY: none
270
  before_script:
271
    - *define-retry
272
    - *conda-init
273
    - conda config --set channel_priority strict
274
  script:
275
    - PACKAGE=${CI_JOB_NAME%%:*}
276
    - cd ${PACKAGE}/
277
    # render clobber YAML file to use our tarball
278
    - TARBALL=$(ls -t1 ${PACKAGE}-*.tar.* | head -n1 | xargs readlink -f)
279
    - SHA256=$(openssl dgst -r -sha256 $TARBALL | cut -d\  -f1)
280
281
    - tar -xf ${TARBALL} --wildcards ${PACKAGE}-*/conda/ --strip-components=1
    - sed 's|@TARBALL@|'${TARBALL}'|g' conda/meta.yaml.in > conda/meta.yaml
282
    - sed -i 's|@SHA256@|'${SHA256}'|g' conda/meta.yaml
283
284
285
286
287
    # create a feedstock from the conda recipe
    - git config --global user.name "${GITLAB_USER_NAME}"
    - git config --global user.email "${GITLAB_USER_EMAIL}"
    - conda smithy init conda/ --feedstock-directory ${PACKAGE}-feedstock
    - cd ${PACKAGE}-feedstock
288
    - retry conda smithy regenerate --no-check-uptodate
289
    - git ls-files
290
    # build packages (preferring local packages over everything)
291
292
293
    # note: we use a variable and `eval` to get around the
    #       _pyversion string having spaces in it
    - cmd="conda build
294
          recipe/
Duncan Macleod's avatar
Duncan Macleod committed
295
          --dirty
296
297
          --error-overlinking
          --error-overdepending
Duncan Macleod's avatar
Duncan Macleod committed
298
          --keep-old-work
299
          --no-anaconda-upload
300
          --use-channeldata
301
          --use-local
302
          --variant-config-files .ci_support/${CONDA_CONFIG}.yaml
303
304
305
306
307
308
309
310
          ${CONDA_BUILD_ARGS}"
    # if not a nightly or web-triggered pipeline, just build one python version
    - if [ ${CI_PIPELINE_SOURCE} != "schedule" ] && [ ${CI_PIPELINE_SOURCE} != "web" ]; then
         _pyversion=$(python -c "import yaml; conf = yaml.load(open('.ci_support/${CONDA_CONFIG}.yaml'), Loader=yaml.SafeLoader); print(conf['python'][-1])");
         cmd="${cmd} --python \"${_pyversion}\"";
      fi
    # run the build
    - eval $cmd
311
312
313
  artifacts:
    expire_in: 18h
    paths:
314
315
      - "conda/bld/linux-64/${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/${CI_JOB_NAME%%:*}-*.tar.bz2"
316
317
      - "conda/bld/linux-64/lib${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/lib${CI_JOB_NAME%%:*}-*.tar.bz2"
318
319
      - "conda/bld/linux-64/python-${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/linux-64/python-${CI_JOB_NAME%%:*}-*.tar.bz2"
320
321
322
323
324
325
      - "conda/bld/broken/${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/${CI_JOB_NAME%%:*}-*.tar.bz2"
      - "conda/bld/broken/lib${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/lib${CI_JOB_NAME%%:*}-*.tar.bz2"
      - "conda/bld/broken/python-${CI_JOB_NAME%%:*}-*.conda"
      - "conda/bld/broken/python-${CI_JOB_NAME%%:*}-*.tar.bz2"
326
327
328
329
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/config.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/test-suite.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/python/test-suite.log"
      - "conda/bld/${CI_JOB_NAME%%:*}-*/work/*/*/*/*/test-suite.log"
330
      - "${CI_JOB_NAME%%:*}/${CI_JOB_NAME%%:*}-feedstock/"
Duncan Macleod's avatar
Duncan Macleod committed
331
    reports:
Duncan Macleod's avatar
Duncan Macleod committed
332
      junit:
333
334
        - "conda/bld/${CI_JOB_NAME%%:*}-*/work/_build*/test/python/junit*.xml"
        - "conda/bld/${CI_JOB_NAME%%:*}-*/test_tmp/junit*.xml"
335
    when: always
336
  cache:
337
    key: $CI_JOB_NAME
338
339
    paths:
      - conda/pkgs
340
341
342
343
344
345
346
347
  only:
    refs:
      - pushes
      - schedules
      - tags
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip conda\]/
348

Duncan Macleod's avatar
Duncan Macleod committed
349
350
# -- linting ------------------------------------

351
.lint:
Duncan Macleod's avatar
Duncan Macleod committed
352
  stage: lint
353
  needs: []
354
355
356
357
358
359
360
361
  only:
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip lint\]/

lint:python:
  extends:
    - .lint
  image: python
Duncan Macleod's avatar
Duncan Macleod committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
  before_script:
    - python -m pip install
          "flake8>=3.7.0"
          "flake8-junit-report"
  script:
    - python -m flake8 --output-file flake8.out
  after_script:
    - cat flake8.out
    - python -m junit_conversor flake8.out junit.xml
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/pip
  artifacts:
376
    expire_in: 1month
Duncan Macleod's avatar
Duncan Macleod committed
377
378
379
380
    reports:
      junit: junit.xml
    when: always

Adam Mercer's avatar
Adam Mercer committed
381
lint:coala:
382
383
  extends:
    - .lint
Adam Mercer's avatar
Adam Mercer committed
384
  image: coala/base
385
386
  before_script:
    - pip install coala-json
Adam Mercer's avatar
Adam Mercer committed
387
  script:
Adam Mercer's avatar
Adam Mercer committed
388
389
390
    # run once to show output to console
    - coala --ci || true
    # run again to prepare JSON report for upload
Adam Mercer's avatar
Adam Mercer committed
391
    - coala --ci --json -o output.json
392
393
394
395
396
397
398
399
400
401
402
  after_script:
    - coala-json --junit -f output.json -o junit.xml
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .cache/pip
  artifacts:
    expire_in: 1month
    reports:
      junit: junit.xml
    when: always
Adam Mercer's avatar
Adam Mercer committed
403

404
405
406
# -- documentation ------------------------------

documentation:
407
  image: igwn/lalsuite-dev:buster
408
  stage: documentation
Adam Mercer's avatar
Adam Mercer committed
409
  needs: []
410
411
412
  script:
    - ./00boot
    - ./configure --enable-doxygen --prefix=${LAL_DIR}
Duncan Macleod's avatar
Duncan Macleod committed
413
    - make -j${CPU_COUNT} install-html
414
415
416
417
418
419
420
421
422
    - pushd ${LAL_DIR}/share/doc
    - cp lalsuite/index.html index.html
    - sed -i 's/..\/lal/lal/g' index.html
    - popd
  artifacts:
    expire_in: 18h
    paths:
      - ${LAL_DIR}/share/doc
  only:
423
424
425
426
427
    refs:
      - pushes
      - web
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip docs\]/
428
429

pages:
430
  image: igwn/lalsuite-dev:buster
431
432
433
434
  stage: deploy
  script:
    - ./00boot
    - ./configure --enable-doxygen --prefix=${LAL_DIR}
Duncan Macleod's avatar
Duncan Macleod committed
435
    - make -j${CPU_COUNT} install-html
436
437
438
439
440
    - cp -r ${LAL_DIR}/share/doc public/
    - pushd public
    - cp lalsuite/index.html index.html
    - sed -i 's/..\/lal/lal/g' index.html
    - popd
441
442
443
    - cp coverage.html public/
  needs:
    - coverage
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
  artifacts:
    paths:
      - public
  only:
    - master@lscsoft/lalsuite
  except:
    - pushes
    - web

# -- tarballs -----------------------------------

# this job makes all of the source tarballs up-front using a top-level
# make dist call
#
# to use one (any) of the tarballs in another job, just add "make-dist"
# in `dependencies:`

make-dist:
462
  image: igwn/lalsuite-dev:el7
463
464
465
466
467
468
469
470
471
472
473
474
475
  stage: .pre
  script:
    - ./00boot
    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
    - for subdir in lal lalframe lalmetaio lalsimulation lalburst lalinspiral lalpulsar lalinference lalapps; do
        pushd ${subdir};
        make dist;
        popd;
      done
  artifacts:
    expire_in: 18h
    paths:
      - "*/*.tar.*"
476

477
478
479
480
481
482
483
484
485
486
# just make the lal tarball to release other jobs
make-dist:lal:
  extends:
    - make-dist
  script:
    - pushd lal
    - ./00boot
    - ./configure --enable-swig ${EXTRA_CONFIG_FLAGS}
    - make dist

487
488
489
490
491
# -- lal ----------------------------------------

lal:rpm:
  <<: *rpmbuild
  stage: LAL
492
  needs:
493
    - make-dist:lal
494

495
496
497
lal:deb:
  <<: *debbuild
  stage: LAL
498
  needs:
499
    - make-dist:lal
500

501
lal:conda:fftw:
502
503
  <<: *conda-build
  stage: LAL
504
  needs:
505
    - make-dist:lal
506
507
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
508
    CONDA_CONFIG: "linux_64_fft_implfftw"
509
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
510

511
512
513
514
lal:conda:mkl:
  <<: *conda-build
  stage: LAL
  needs:
515
    - make-dist:lal
516
517
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
518
    CONDA_CONFIG: "linux_64_fft_implmkl"
519
520
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"

521
522
523
524
525
# -- lalframe------------------------------------

lalframe:rpm:
  <<: *rpmbuild
  stage: LALFrame
526
  needs:
527
    - make-dist
528
    - lal:rpm
529

530
531
532
lalframe:deb:
  <<: *debbuild
  stage: LALFrame
533
  needs:
534
    - make-dist
535
    - lal:deb
536

537
538
539
lalframe:conda:
  <<: *conda-build
  stage: LALFrame
540
  needs:
541
    - make-dist
542
    - lal:conda:fftw
543

544
545
546
547
548
# -- lalmetaio ----------------------------------

lalmetaio:rpm:
  <<: *rpmbuild
  stage: LALMetaIO
549
  needs:
550
    - make-dist
551
    - lal:rpm
552

553
554
555
lalmetaio:deb:
  <<: *debbuild
  stage: LALMetaIO
556
  needs:
Duncan Macleod's avatar
Duncan Macleod committed
557
    - make-dist
558
    - lal:deb
Duncan Macleod's avatar
Duncan Macleod committed
559

560
561
562
563
564
lalmetaio:conda:
  <<: *conda-build
  stage: LALMetaIO
  needs:
    - make-dist
565
    - lal:conda:fftw
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587

# -- lalsimulation ------------------------------

lalsimulation:rpm:
  <<: *rpmbuild
  stage: LALSimulation
  needs:
    - make-dist
    - lal:rpm

lalsimulation:deb:
  <<: *debbuild
  stage: LALSimulation
  needs:
    - make-dist
    - lal:deb

lalsimulation:conda:
  <<: *conda-build
  stage: LALSimulation
  needs:
    - make-dist
588
    - lal:conda:fftw
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614

# -- lalburst -----------------------------------

lalburst:rpm:
  <<: *rpmbuild
  stage: LALBurst
  needs:
    - make-dist
    - lal:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm

lalburst:deb:
  <<: *debbuild
  stage: LALBurst
  needs:
    - make-dist
    - lal:deb
    - lalmetaio:deb
    - lalsimulation:deb

lalburst:conda:
  <<: *conda-build
  stage: LALBurst
  needs:
    - make-dist
615
    - lal:conda:fftw
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
    - lalmetaio:conda
    - lalsimulation:conda

# -- lalinspiral --------------------------------

lalinspiral:rpm:
  <<: *rpmbuild
  stage: LALInspiral
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm

lalinspiral:deb:
  <<: *debbuild
  stage: LALInspiral
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb

lalinspiral:conda:
  <<: *conda-build
  stage: LALInspiral
  needs:
    - make-dist
648
    - lal:conda:fftw
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda

# -- lalpulsar ----------------------------------

lalpulsar:rpm:
  <<: *rpmbuild
  stage: LALPulsar
  needs:
    - make-dist
    - lal:rpm

lalpulsar:deb:
  <<: *debbuild
  stage: LALPulsar
  needs:
    - make-dist
    - lal:deb

lalpulsar:conda:
  <<: *conda-build
  stage: LALPulsar
  needs:
    - make-dist
675
    - lal:conda:fftw
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709

# -- lalinference -------------------------------

lalinference:rpm:
  <<: *rpmbuild
  stage: LALInference
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm

lalinference:deb:
  <<: *debbuild
  stage: LALInference
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb

lalinference:conda:
  <<: *conda-build
  stage: LALInference
  needs:
    - make-dist
710
    - lal:conda:fftw
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda
    - lalpulsar:conda
    - lalinspiral:conda

# -- lalapps ------------------------------------

lalapps:rpm:
  <<: *rpmbuild
  stage: LALApps
  needs:
    - make-dist
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm

lalapps:deb:
  <<: *debbuild
  stage: LALApps
  needs:
    - make-dist
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb

lalapps:conda:
  <<: *conda-build
  stage: LALApps
  needs:
    - make-dist
753
    - lal:conda:fftw
754
755
756
757
758
759
760
    - lalframe:conda
    - lalmetaio:conda
    - lalsimulation:conda
    - lalburst:conda
    - lalpulsar:conda
    - lalinspiral:conda
    - lalinference:conda
761
762
  variables:
    CONDA_BLD_PATH: "${CI_PROJECT_DIR}/conda/bld"
763
    CONDA_CONFIG: "linux_64_python3.7.____cpython"
764
    CONDA_PKGS_DIRS: "${CI_PROJECT_DIR}/conda/pkgs"
765
766
767

# -- integration tests --------------------------

768
769
770
771
772
773
.integration-tests: &integration-tests
  stage: integration tests
  only:
    variables:
      - $CI_COMMIT_MESSAGE !~ /\[skip integration\]/

774
lalinference_testjob:
775
  <<: *yum-cache
776
777
  extends:
    - .integration-tests
778
  image: igwn/base:el7-testing
779
780
  before_script:
    - *yum-update
781
782
783
    - ${YUM} install
          git
          git-lfs
784
    - ${YUM} install rpmbuild/RPMS/x86_64/*.rpm
785
786
787
788
789
790
791
792
793
794
795
796
797
  script:
    - git lfs clone https://git.ligo.org/lscsoft/ROQ_data --include "**/params.dat,*/4s/**"
    - bash lalinference/test/lalinference_nestedSampling_integration_test.sh
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
  only:
798
799
800
    refs:
      - schedules
      - web
801
802
803
804
805
  artifacts:
    expire_in: 18h
    paths:
      - example/
    when: always
806

Adam Mercer's avatar
Adam Mercer committed
807
top-level:
808
  image: igwn/lalsuite-dev:el7-testing
Adam Mercer's avatar
Adam Mercer committed
809
  needs: []
810
  <<: *top-level-build
811
812
  extends:
    - .integration-tests
813
  only:
814
815
816
817
    refs:
      - pushes
      - schedules
      - web
818

819
820
# Note: adapted from https://git.ligo.org/lscsoft/ligo.skymap.
coverage:
821
822
  extends:
    - .integration-tests
823
  image: igwn/lalsuite-dev-gcc:8
824
825
826
827
828
829
830
831
  needs: []
  script:
    - apt-get -y install --no-install-recommends python3-pip
    - python3 -m pip install coverage gcovr pycobertura
    - ./00boot
    # Configure for C code coverage measurement with gcc
    - ./configure --enable-swig-python CFLAGS="--coverage -fprofile-abs-path" PYTHON=python3
    # Run unit tests with Python scripts instrumented by coverage.py
Duncan Macleod's avatar
Duncan Macleod committed
832
    - make -j${CPU_COUNT} check COVERAGE_FILE="$(pwd)/.coverage" PY_LOG_FLAGS="-m coverage run --source \"$(pwd)\" --omit \*/test/\* -p"
833
    # Output C coverage data in Cobertura format
Duncan Macleod's avatar
Duncan Macleod committed
834
    - gcovr -j${CPU_COUNT} --exclude-directories test --xml c-coverage.xml
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
    # Combine and output Python coverage data in Cobertura format
    - coverage combine
    - coverage xml -o py-coverage.xml
    # Merge coverage reports. They're just XML, after all.
    - |
      python3 - <<EOF
      import lxml.etree
      import copy

      doc1 = lxml.etree.parse('py-coverage.xml')
      doc2 = lxml.etree.parse('c-coverage.xml')
      root1 = doc1.getroot()
      root2 = doc2.getroot()
      root1.attrib['lines-covered'] = str(
          int(root1.attrib['lines-covered']) +
          int(root2.attrib['lines-covered']))
      root1.attrib['lines-valid'] = str(
          int(root1.attrib['lines-valid']) +
          int(root2.attrib['lines-valid']))
      try:
          root1.attrib['line-rate'] = str(
              int(root1.attrib['lines-covered']) /
              int(root1.attrib['lines-valid']))
      except ZeroDivisionError:
          root1.attrib['line-rate'] = '0'
      root1.attrib['branches-covered'] = str(
          int(root1.attrib['branches-covered']) +
          int(root2.attrib['branches-covered']))
      root1.attrib['branches-valid'] = str(
          int(root1.attrib['branches-valid']) +
          int(root2.attrib['branches-valid']))
      try:
          root1.attrib['branch-rate'] = str(
              int(root1.attrib['branches-covered']) /
              int(root1.attrib['branches-valid']))
      except ZeroDivisionError:
          root1.attrib['branch-rate'] = '0'
      packages = root1.find('./packages')
      packages.extend(root2.iterfind('./packages/package'))
      doc1.write('coverage.xml')
      EOF
    # Write HTML coverage report and show coverage summary.
    - pycobertura show coverage.xml -f html -o coverage.html
    - pycobertura show coverage.xml | tail -n 1
  coverage: '/TOTAL\s+.*\s+([\d\.]+)%/'
  artifacts:
    paths:
      - coverage.html
    reports:
      cobertura: coverage.xml
  only:
886
887
888
889
    refs:
      - pushes
      - schedules
      - web
890

891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
# -- docker -------------------------------------

# build a nightly container from the RPMs
docker:nightly:el7:
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add RPMs to directory to pass to docker
    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
    - rm -rf rpmbuild*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker build --pull -t $CI_REGISTRY_IMAGE/nightly:el7 --file .gitlab-ci-el7.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/nightly:el7
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
    - lalapps:rpm
  only:
    - schedules
918

919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
# build a tagged container from the rpms
docker:tags:el7:
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add RPMs to directory to pass to docker
    - mkdir rpms && mv rpmbuild/RPMS/x86_64/*.rpm rpms
    - rm -rf rpmbuild*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker build --pull -t $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:el7 --file .gitlab-ci-el7.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:el7
  needs:
    - lal:rpm
    - lalframe:rpm
    - lalmetaio:rpm
    - lalsimulation:rpm
    - lalburst:rpm
    - lalpulsar:rpm
    - lalinspiral:rpm
    - lalinference:rpm
    - lalapps:rpm
  only:
    - /^lalsuite-v.*$/

# build a nightly container from the debian packages
946
docker:nightly:buster:
947
948
949
950
951
952
953
954
955
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add deb packages to directory to pass to docker
    - mkdir debs && mv *.deb debs
    - rm *.changes *.dsc *.orig.*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
956
957
    - docker build --pull -t $CI_REGISTRY_IMAGE/nightly:buster --file .gitlab-ci-buster.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/nightly:buster
958
959
960
961
962
963
964
965
966
967
968
969
970
971
  needs:
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb
    - lalapps:deb
  only:
    - schedules

# build a tagged container from the debian packages
972
docker:tags:buster:
973
974
975
976
977
978
979
980
981
  stage: docker
  before_script: []
  script:
    - set -x  # print commands as they are run (helps debugging)
    # add deb packages to directory to pass to docker
    - mkdir debs && mv *.deb debs
    - rm *.changes *.dsc *.orig.*
    # build container and push to registry
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
982
983
    - docker build --pull -t $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:buster --file .gitlab-ci-buster.Dockerfile .
    - docker push $CI_REGISTRY_IMAGE/$CI_COMMIT_TAG:buster
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
  needs:
    - lal:deb
    - lalframe:deb
    - lalmetaio:deb
    - lalsimulation:deb
    - lalburst:deb
    - lalpulsar:deb
    - lalinspiral:deb
    - lalinference:deb
    - lalapps:deb
  only:
    - /^lalsuite-v.*$/

# -- nightly ------------------------------------

# -- compilers
Adam Mercer's avatar
Adam Mercer committed
1000

For faster browsing, not all history is shown. View entire blame