.gitlab-ci.yml 9.68 KB
Newer Older
1
include:
Leo Pound Singer's avatar
Leo Pound Singer committed
2 3
  - project: emfollow/gitlab-ci-templates
    file: lint.yml
4 5
  - project: emfollow/gitlab-ci-templates
    file: sccb.yml
Leo Pound Singer's avatar
Leo Pound Singer committed
6 7
  - project: emfollow/gitlab-ci-templates
    file: ssh.yml
8

9
variables:
10
  # Allow Celery to run as root, because everything runs as root under Docker.
11
  C_FORCE_ROOT: 'true'
12 13 14 15
  # Checking out the source is the exception rather than the rule because
  # most of our CI jobs run from tarballs or wheels or don't require the source
  # at all.
  GIT_STRATEGY: none
16

17 18 19 20 21 22 23 24
.in-tmpdir: &in-tmpdir
  before_script:
    - WORKING_DIRECTORY="$(mktemp -d)"
    - cd "${WORKING_DIRECTORY}"
  after_script:
    - cd "${CI_PROJECT_DIR}"
    - rm -rf "${WORKING_DIRECTORY}"

Leo Pound Singer's avatar
Leo Pound Singer committed
25
stages:
26
  - dist
Leo Pound Singer's avatar
Leo Pound Singer committed
27
  - test
Leo Pound Singer's avatar
Leo Pound Singer committed
28 29
  - deploy

30 31
# Build source distribution
sdist:
Leo Pound Singer's avatar
Leo Pound Singer committed
32
  image: python:slim
33
  stage: dist
34 35
  variables:
    GIT_STRATEGY: fetch
36 37 38
  before_script:
    # Versioneer will need access to the git tool.
    - apt-get -q update
Leo Pound Singer's avatar
Leo Pound Singer committed
39
    - apt-get -yq install --no-install-recommends git
Leo Pound Singer's avatar
Leo Pound Singer committed
40
  script:
41
    - python setup.py sdist
42
    - mv dist/* .
Leo Pound Singer's avatar
Leo Pound Singer committed
43 44
  artifacts:
    paths:
45
      - '*.tar.gz'
46 47 48

# Build binary distribution
bdist:
Leo Pound Singer's avatar
Leo Pound Singer committed
49
  image: python:slim
50
  stage: test
51
  <<: *in-tmpdir
52
  script:
53 54
    - tar --strip-components 1 -xf ${CI_PROJECT_DIR}/*.tar.*
    - python setup.py bdist_wheel --dist-dir ${CI_PROJECT_DIR}
55 56 57 58
  dependencies:
    - sdist
  artifacts:
    paths:
Leo Pound Singer's avatar
Leo Pound Singer committed
59 60
      - '*.whl'

61
# Build Docker container for dependencies
Leo Pound Singer's avatar
Leo Pound Singer committed
62
.dependencies: &dependencies
63
  stage: dist
64
  variables:
65
    GIT_STRATEGY: fetch
66 67
    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_NAME
  script:
68
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
69 70
    - |
      cat <<EOF > Dockerfile
Leo Pound Singer's avatar
Leo Pound Singer committed
71
      FROM python:${CI_JOB_NAME#*python}
72 73 74
      COPY requirements.txt .
      RUN pip --no-cache-dir install -r requirements.txt
      RUN rm -f requirements.txt
75 76 77
      EOF
    - docker build -t $IMAGE_TAG .
    - docker push $IMAGE_TAG
78
    - if [ "${CI_COMMIT_TAG:0:1}" = "v" ]; then docker tag $IMAGE_TAG ${IMAGE_TAG%:*}:latest; docker push ${IMAGE_TAG%:*}:latest; fi
79 80 81
  only:
    changes:
      - requirements.txt
Leo Pound Singer's avatar
Leo Pound Singer committed
82 83
dependencies/python3.7:
  <<: *dependencies
Leo Pound Singer's avatar
Leo Pound Singer committed
84 85
dependencies/python3.8:
  <<: *dependencies
86 87

# Run unit tests and coverage measurement
Leo Pound Singer's avatar
Leo Pound Singer committed
88
.test: &test
Leo Pound Singer's avatar
Leo Pound Singer committed
89
  stage: test
Leo Pound Singer's avatar
Leo Pound Singer committed
90
  coverage: '/^TOTAL\s+.*\s+(\d+\.?\d*)%/'
91
  <<: *in-tmpdir
92
  script:
93 94
    - apt-get update -qq && apt-get install -y -qq redis-server
    - tar --strip-components 1 -xf ${CI_PROJECT_DIR}/*.tar.*
95
    - pip install pytest-cov
96
    - python setup.py test --addopts="-vv --cov --cov-report=html:${CI_PROJECT_DIR}/htmlcov --cov-report=term --junit-xml=${CI_PROJECT_DIR}/junit.xml"
97
  dependencies:
98
    - sdist
Leo Pound Singer's avatar
Leo Pound Singer committed
99 100 101
  artifacts:
    paths:
      - htmlcov/
102 103
    reports:
      junit: junit.xml
Leo Pound Singer's avatar
Leo Pound Singer committed
104 105 106
test/python3.7:
  image: $CI_REGISTRY_IMAGE/dependencies/python3.7:$CI_COMMIT_REF_NAME
  <<: *test
Leo Pound Singer's avatar
Leo Pound Singer committed
107 108 109
test/python3.8:
  image: $CI_REGISTRY_IMAGE/dependencies/python3.8:$CI_COMMIT_REF_NAME
  <<: *test
Leo Pound Singer's avatar
Leo Pound Singer committed
110

111
lint:
Leo Pound Singer's avatar
Leo Pound Singer committed
112
  image: $CI_REGISTRY_IMAGE/dependencies/python3.8:$CI_COMMIT_REF_NAME
113
  stage: test
114 115
  variables:
    GIT_STRATEGY: fetch
116
  dependencies:
117
    - sdist
118

Leo Pound Singer's avatar
Leo Pound Singer committed
119 120 121 122 123
associate commits in Sentry:
  stage: test
  dependencies:
    - sdist
  variables:
Leo Pound Singer's avatar
Leo Pound Singer committed
124
    GIT_STRATEGY: fetch
Leo Pound Singer's avatar
Leo Pound Singer committed
125 126 127
  only:
    - master@emfollow/gwcelery
    - tags@emfollow/gwcelery
Leo Pound Singer's avatar
Leo Pound Singer committed
128 129 130
  image:
    name: getsentry/sentry-cli
    entrypoint: [""]
Leo Pound Singer's avatar
Leo Pound Singer committed
131
  script:
Leo Pound Singer's avatar
Leo Pound Singer committed
132
    - SENTRY_VERSION=$(echo *.tar.* | sed 's/\.tar\..*//')
Leo Pound Singer's avatar
Leo Pound Singer committed
133 134
    - sentry-cli releases new ${SENTRY_VERSION}
    - sentry-cli releases set-commits --auto ${SENTRY_VERSION}
Leo Pound Singer's avatar
Leo Pound Singer committed
135

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
# Build docker container for application itself
.docker: &docker
  stage: deploy
  script:
    - IMAGE_TAG=$CI_REGISTRY_IMAGE/${CI_JOB_NAME#*/}:$CI_COMMIT_REF_NAME
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - |
      cat <<EOF > Dockerfile
      FROM $CI_REGISTRY_IMAGE/dependencies/${CI_JOB_NAME#*/}:$CI_COMMIT_REF_NAME
      COPY *.whl .
      RUN pip install *.whl
      RUN rm -f *.whl
      USER nobody
      WORKDIR /tmp
      ENTRYPOINT ["gwcelery"]
      EOF
    - docker build -t $IMAGE_TAG .
    - docker push $IMAGE_TAG
    - if [ "${CI_COMMIT_TAG:0:1}" = "v" ]; then docker tag $IMAGE_TAG ${IMAGE_TAG%:*}:latest; docker push ${IMAGE_TAG%:*}:latest; fi
  dependencies:
    - bdist
Leo Pound Singer's avatar
Leo Pound Singer committed
157 158
docker/python3.7:
  <<: *docker
Leo Pound Singer's avatar
Leo Pound Singer committed
159 160
docker/python3.8:
  <<: *docker
161

162
# Generate documentation
163
doc:
Leo Pound Singer's avatar
Leo Pound Singer committed
164
  image: python:slim
Leo Pound Singer's avatar
Leo Pound Singer committed
165
  stage: test
166
  <<: *in-tmpdir
167
  script:
168
    - apt-get update -qq && apt-get install -y -qq graphviz
Leo Pound Singer's avatar
Leo Pound Singer committed
169
    - tar --strip-components 1 -xf ${CI_PROJECT_DIR}/*.tar.*
Leo Pound Singer's avatar
Leo Pound Singer committed
170
    - pip install -r docs-requirements.txt
171
    - python setup.py build_sphinx
172
  dependencies:
173
    - sdist
174

175
# SCCB request
176
sccb:
177 178
  stage: deploy

179
# Publish coverage
Leo Pound Singer's avatar
Leo Pound Singer committed
180 181 182
pages:
  stage: deploy
  script:
Leo Pound Singer's avatar
Leo Pound Singer committed
183
    - mv htmlcov public/
Leo Pound Singer's avatar
Leo Pound Singer committed
184 185 186 187 188 189
  artifacts:
    paths:
      - public
    expire_in: 30 days
  only:
    - master
190
  dependencies:
191
    - test/python3.7
Leo Pound Singer's avatar
Leo Pound Singer committed
192 193 194 195

# Upload package to PyPI
pypi:
  stage: deploy
Leo Pound Singer's avatar
Leo Pound Singer committed
196
  image: python:slim
Leo Pound Singer's avatar
Leo Pound Singer committed
197 198 199 200
  script:
    - pip install twine
    - twine upload *.tar.* *.whl
  dependencies:
201 202
    - sdist
    - bdist
Leo Pound Singer's avatar
Leo Pound Singer committed
203 204
  only:
    - tags@emfollow/gwcelery
205

Leo Pound Singer's avatar
Leo Pound Singer committed
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
# Create a release in GitLab
release:
  stage: deploy
  image: python
  variables:
    GIT_STRATEGY: fetch
  script:
    - PACKAGE_NAME="$(python setup.py --name)"
    - PACKAGE_VERSION="$(python setup.py --version)"
    - CHANGES_FILENAME="$(echo CHANGES.*)"
    - |
      tee json <<EOF
      {
        "name": "${PACKAGE_VERSION}",
        "tag_name": "${CI_COMMIT_TAG}",
        "description": "Version ${PACKAGE_VERSION}",
        "assets": {
          "links": [
            {
              "name": "PyPI",
              "url": "https://pypi.org/project/${PACKAGE_NAME}/${PACKAGE_VERSION}/"
            },
            {
              "name": "Change log",
              "url": "${CI_PROJECT_URL}/blob/v${VERSION}/${CHANGES_FILENAME}"
            },
            {
              "name": "Documentation",
              "url": "https://${PACKAGE_NAME}.readthedocs.io/en/v${PACKAGE_VERSION}/"
            }
          ]
        }
      }
      EOF
    - |
      if [ -n "$CI_COMMIT_TAG" ]
      then
        curl --silent --show-error --fail \
        --request POST --header "Private-Token: $GITLAB_ACCESS_TOKEN" \
        --header "Content-Type: application/json" \
Leo Pound Singer's avatar
Leo Pound Singer committed
246
        --data "@json" \
Leo Pound Singer's avatar
Leo Pound Singer committed
247
        "$CI_API_V4_URL/projects/$CI_PROJECT_ID/releases"
Leo Pound Singer's avatar
Leo Pound Singer committed
248
      fi
Leo Pound Singer's avatar
Leo Pound Singer committed
249

250
.deploy-common: &deploy-common
Leo Pound Singer's avatar
Leo Pound Singer committed
251
  extends: .ssh
252 253
  stage: deploy
  only:
254
    - branches@emfollow/gwcelery
255
    - tags@emfollow/gwcelery
256
  dependencies: []
257 258 259

# Continuous deployment.
# Note that all deployments except playground/CIT are manual.
Leo Pound Singer's avatar
Leo Pound Singer committed
260

261 262
.deploy-start: &deploy-start
  <<: *deploy-common
Leo Pound Singer's avatar
Leo Pound Singer committed
263 264 265
  script:
    # Update the repository.
    - |
266
      gsissh -T $REMOTE_HOST bash <<EOF
Leo Pound Singer's avatar
Leo Pound Singer committed
267
      set -e
Leo Pound Singer's avatar
Leo Pound Singer committed
268
      if [ ! -d .git ]; then git init; fi
269
      git fetch --tags ${CI_REPOSITORY_URL} $CI_COMMIT_SHA
270
      git checkout -f $CI_COMMIT_SHA
Leo Pound Singer's avatar
Leo Pound Singer committed
271 272 273
      EOF
    # Reinstall and start (new session to reload bashrc).
    - |
274
      gsissh -T $REMOTE_HOST bash <<EOF
Leo Pound Singer's avatar
Leo Pound Singer committed
275
      . .bashrc
Leo Pound Singer's avatar
Leo Pound Singer committed
276
      set -e
Leo Pound Singer's avatar
Leo Pound Singer committed
277
      python -m pip install --upgrade pipenv pip setuptools
278
      pipenv install --system --deploy
279 280
      # FIXME: remove pip uninstall temporary directories left behind due to
      # bug with pip on NFS. See https://github.com/pypa/pip/issues/6327
281 282
      chmod -R ug+rwx .local/lib/python3.7/site-packages/~* || true
      rm -rf .local/lib/python3.7/site-packages/~* || true
Leo Pound Singer's avatar
Leo Pound Singer committed
283 284
      gwcelery condor resubmit
      EOF
285 286 287 288

.deploy-stop: &deploy-stop
  <<: *deploy-common
  script: |
289
    gsissh -T $REMOTE_HOST bash <<EOF
Leo Pound Singer's avatar
Leo Pound Singer committed
290
    . .bashrc
Leo Pound Singer's avatar
Leo Pound Singer committed
291
    set -e
292
    gwcelery condor rm
293 294
    EOF

295
deploy to production at CIT:
296
  <<: *deploy-start
297
  when: manual
298
  environment:
299 300
    name: production/CIT
    on_stop: stop deployment on production at CIT
301
    url: https://emfollow.ligo.caltech.edu/flower
302 303
  variables:
    REMOTE_HOST: emfollow.ligo.caltech.edu
304

305
deploy to production at LHO:
306
  <<: *deploy-start
307
  when: manual
308
  environment:
309 310
    name: production/LHO
    on_stop: stop deployment on production at LHO
311
    url: https://emfollow.ligo-wa.caltech.edu/flower
312 313
  variables:
    REMOTE_HOST: emfollow.ligo-wa.caltech.edu
314 315 316 317 318 319

deploy to playground at CIT:
  <<: *deploy-start
  environment:
    name: playground/CIT
    on_stop: stop deployment on playground at CIT
320
    url: https://emfollow-playground.ligo.caltech.edu/flower
321 322
  variables:
    REMOTE_HOST: emfollow-playground.ligo.caltech.edu
323

324 325
deploy to playground at LHO:
  <<: *deploy-start
326
  when: manual
327 328 329
  environment:
    name: playground/LHO
    on_stop: stop deployment on playground at LHO
330
    url: https://emfollow-playground.ligo-wa.caltech.edu/flower
331 332
  variables:
    REMOTE_HOST: emfollow-playground.ligo-wa.caltech.edu
333

334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
deploy to test at CIT:
  <<: *deploy-start
  when: manual
  environment:
    name: test/CIT
    on_stop: stop deployment on test at CIT
    url: https://emfollow-test.ligo.caltech.edu/flower
  variables:
    REMOTE_HOST: emfollow-test.ligo.caltech.edu

deploy to test at LHO:
  <<: *deploy-start
  when: manual
  environment:
    name: test/LHO
    on_stop: stop deployment on test at LHO
    url: https://emfollow-test.ligo-wa.caltech.edu/flower
  variables:
    REMOTE_HOST: emfollow-test.ligo-wa.caltech.edu

354 355
stop deployment on production at CIT:
  <<: *deploy-stop
356
  when: manual
357 358 359 360 361 362
  environment:
    name: production/CIT
    action: stop

stop deployment on production at LHO:
  <<: *deploy-stop
363
  when: manual
364 365 366 367 368
  environment:
    name: production/LHO
    action: stop

stop deployment on playground at CIT:
369
  <<: *deploy-stop
370
  when: manual
371
  environment:
372
    name: playground/CIT
373 374
    action: stop

375
stop deployment on playground at LHO:
376
  <<: *deploy-stop
377
  when: manual
378
  environment:
379
    name: playground/LHO
380
    action: stop
381 382 383 384 385 386 387 388 389 390 391 392 393 394

stop deployment on test at CIT:
  <<: *deploy-stop
  when: manual
  environment:
    name: test/CIT
    action: stop

stop deployment on test at LHO:
  <<: *deploy-stop
  when: manual
  environment:
    name: test/LHO
    action: stop