...
 
Commits (90)
**.swp
docs/_build
__pycache__
build
docs/environments/environment*.rst
---
image: docker:latest
stages:
- test
- build
- docs
- deploy
variables:
CVMFS_BASE_DIR: "/cvmfs/ligo-containers.opensciencegrid.org/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}"
DOCKER_DRIVER: overlay
TAG: $CI_REGISTRY_IMAGE:latest
NIGHTLY: $CI_REGISTRY_IMAGE:nightly
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
build:
test-macos:
tags:
- macos_highsierra
stage: test
variables:
INSTALL_PATH: "${CI_PROJECT_DIR}/miniconda"
only:
- master
- merge_requests
- schedules
before_script: []
script:
- curl -o ./miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
- /bin/bash miniconda.sh -b -p ${INSTALL_PATH}
- source ${INSTALL_PATH}/etc/profile.d/conda.sh
- conda update --yes conda
- conda env create -f environment-py27.yml
- conda env create -f environment-py36.yml
- conda env create -f environment-py37.yml
test-linux:
stage: test
only:
- merge_requests
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build --no-cache --pull -t $TAG --build-arg INSTALL_PATH=/cvmfs/ligo-containers.opensciencegrid.org/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/latest .
- docker build --pull --tag ${CI_REGISTRY_IMAGE}:test --build-arg INSTALL_PATH=/opt/conda .
build-latest:
stage: build
only:
- master
script:
- docker build --no-cache --pull -t $TAG --build-arg INSTALL_PATH=${CVMFS_BASE_DIR}/latest .
- docker push $TAG
build-nightly:
stage: build
only:
- schedules
script:
- docker build --no-cache --pull -t $NIGHTLY -f Dockerfile-singularity .
- docker push $NIGHTLY
# -- documentation
docs:
stage: docs
dependencies:
- build-latest
only:
- schedules
before_script:
# install python
- apk add python3
- export LSCSOFT_CONDA_DOCKER_IMAGE="${NIGHTLY}"
- python3 -m pip install -r docs/requirements.txt
script:
- cd docs
- python3 -m sphinx -M html . ../build/sphinx -vv
# remove index.html in links because duncan is _really_ pedantic
- find ../build/sphinx/html -name "*.html" -type f -exec sed -i "s/index.html//g" {} +
artifacts:
paths:
- build/sphinx/html
cache:
paths:
- .cache/pip
pages:
stage: deploy
dependencies:
- docs
only:
- schedules
script:
- mv build/sphinx/html public
artifacts:
paths:
- public
.gitlab.yml thomas.downes@ligo.org
Dockerfile* thomas.downes@ligo.org
entrypoint thomas.downes@ligo.org
LICENSE thomas.downes@ligo.org
/docs/ duncan.macleod@ligo.org
<!--
Please read this!
Before opening a new issue, make sure to search for keywords in the issues filtered by the `bug` label
https://git.ligo.org/lscsoft/conda/issues?label_name%5B%5D=bug
and verify the issue you're about to submit isn't a duplicate.
-->
### Summary
(Summarize the bug encountered concisely)
### Steps to reproduce
(How one can reproduce the issue - this is very important)
### Environment
<details>
<summary>Environment (<code>conda list</code>):</summary>
```
$ conda list
```
</details>
<br/>
<details>
<summary>Details about <code>conda</code> and system ( <code>conda info</code> ):</summary>
```
$ conda info
```
</details>
### What is the current *bug* behavior?
(What actually happens)
### What is the expected *correct* behavior?
(What you should see instead)
### Possible fixes
(If you can, link to the line of code that might be responsible for the problem)
/label ~bug
<!--
Please complete this form as fully as you can, so that your request can be evaluated in a timely manner.
The following restrictions will be placed on all requests:
- the package is available (with the same name) on both Linux and macOS
- the package must be available on the conda-forge conda channel
-->
#### Details
- **Package name:** <!-- insert package name -->
- **Version:** <!-- insert specific package version -->
#### Description
<!--
Insert reasoning for request here
In particular, include a description of the requested package, and why is is required.
Please also try and detail what downstream packages/groups may be impacted by your request, either negatively or positively.
**If this is a package update request, please link to the relase notes or changelog for the updated version**
-->
#### Python versions
Please select the Python versions appropriate for this package (deselect those that are not required):
- [x] 2.7
- [x] 3.6
- [x] 3.7
#### Request type
Please select all of the following that apply:
- [ ] this is a new package
- [ ] this is a backwards-compatible update
- [ ] this is a backwards-incompatible update [API/ABI changes]
/label ~request
......@@ -24,22 +24,23 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86
/bin/bash /root/anaconda.sh -b -p $INSTALL_PATH && \
rm -f ~/anaconda.sh && \
. $INSTALL_PATH/etc/profile.d/conda.sh && \
conda update conda
COPY environment.yml /tmp
RUN conda env create -f /tmp/environment.yml
COPY environment-gcc7.yml /tmp
RUN conda env create -f /tmp/environment-gcc7.yml
COPY environment-py35.yml /tmp
RUN conda env create -f /tmp/environment-py35.yml
conda install --yes \
conda \
conda-build \
conda-verify && \
conda config --system --prepend channels conda-forge
COPY environment-py27.yml /tmp
RUN conda env create -f /tmp/environment-py27.yml
COPY environment-py36.yml /tmp
RUN conda env create -f /tmp/environment-py36.yml
COPY environment-py37.yml /tmp
RUN conda env create -f /tmp/environment-py37.yml
RUN conda clean --all --yes
FROM scratch
ARG INSTALL_PATH
ARG INSTALL_PATH=/cvmfs/ligo-containers.opensciencegrid.org/lscsoft/conda/latest/
COPY --from=0 $INSTALL_PATH /
FROM debian:stretch
ARG INSTALL_PATH=/opt/conda
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV PATH $INSTALL_PATH/bin:$PATH
RUN apt-get update --fix-missing && \
apt-get install --yes \
wget \
bzip2 \
ca-certificates \
libglib2.0-0 \
libxext6 \
libsm6 \
libxrender1 \
git \
mercurial \
subversion && \
apt-get clean
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /root/anaconda.sh && \
mkdir -p $(dirname $INSTALL_PATH) && \
/bin/bash /root/anaconda.sh -b -p $INSTALL_PATH && \
rm -f ~/anaconda.sh && \
. $INSTALL_PATH/etc/profile.d/conda.sh && \
conda install --yes \
conda \
conda-build \
conda-verify && \
conda config --system --prepend channels conda-forge
COPY environment-py27.yml /tmp
RUN conda env create -f /tmp/environment-py27.yml
COPY environment-py36.yml /tmp
RUN conda env create -f /tmp/environment-py36.yml
COPY environment-py37.yml /tmp
RUN conda env create -f /tmp/environment-py37.yml
RUN conda clean --all --yes
RUN cp $INSTALL_PATH/etc/profile.d/conda.sh /etc/profile.d
COPY entrypoint /bin
RUN chmod 0755 /bin/entrypoint
ENTRYPOINT [ "entrypoint" ]
CMD [ "ligo-py37", "lal_version" ]
*************
LSCSoft Conda
*************
|MIT license| |Build status|
.. |MIT license| image:: https://img.shields.io/badge/License-MIT-blue.svg
:target: https://opensource.org/licenses/MIT
.. |Build status| image:: https://git.ligo.org/lscsoft/conda/badges/master/pipeline.svg
:target: https://git.ligo.org/lscsoft/conda/commits/master
LSCSoft Conda is a programme to manage and distribute software
used by the LIGO Scientific Collaboration and partner groups
using the `conda <https://conda.io>`_ package manager, and the
`conda-forge <https://conda-forge.org>`_ community.
Quickstart
==========
**Use a pre-built environment (linux only):**
1. `Install and configure CVMFS <https://wiki.ligo.org/Computing/CvmfsUser>`_
2. Configure ``conda``:
.. code-block:: bash
source /cvmfs/ligo-containers.opensciencegrid.org/lscsoft/conda/latest/etc/profile.d/conda.sh
3. Activate an environment:
.. code-block:: bash
conda activate ligo-py37
For more details see `Reference environments <https://docs.ligo.org/lscsoft/conda/environments/>`_.
**Or install miniconda yourself:**
1. `Install Miniconda <https://conda.io/projects/conda/en/latest/user-guide/install/index.html>`_.
2. Add the conda-forge channel:
.. code-block:: bash
conda config --add channels conda-forge
3. Create an environment full of packages:
.. code-block:: bash
conda create --name myenv python=3.7 gwpy python-lal ...
For either solution, see `Tips and tricks <https://docs.ligo.org/lscsoft/conda/tips/>`_ for
more useful hints.
Contributing
============
If you would like to improve LSCSoft Conda, please consider one of the
following actions:
- `Report a problem <https://git.ligo.org/lscsoft/conda/issues/new?issuable_template=Bug>`_
- `Request a new/updated package <https://git.ligo.org/lscsoft/conda/issues/new?issuable_template=Request>`_
#!/bin/sh
if [ $(uname -s) = "Darwin" ]; then
DEFAULT_SDK_DIR=~/SDKs/MacOSX10.9.sdk
DEFAULT_SDK_DIR=/opt/MacOSX10.9.sdk
if [ -d $DEFAULT_SDK_DIR ]; then
export CONDA_BUILD_SYSROOT=$DEFAULT_SDK_DIR
else
......
img.badge {
max-width: none;
}
.. _lscsoft-conda-compiling:
*************************************
Compiling code in a Conda environment
*************************************
.. note::
The :ref:`lscsoft-conda-pre-built-environments` should contain all
of the packages needed to build most things, if packages are missing
please `open a Request ticket
<https://git.ligo.org/lscsoft/conda/issues/new?issuable_template=Request>`_
to request their addition.
To compile packages without using ``conda build`` you will need to build
a conda environment that contains the necessary packages.
The only tricky thing about this is installing the compilers,
`read this
<https://conda.io/docs/user-guide/tasks/build-packages/compiler-tools.html>`__.
You will need to **activate** the environment properly so as to have the
compiler tools set the necessary environment variables (``CC`` etc).
.. note::
There is now a suite of conda-forge packages designed to provide a
compiler for your architecture.
See the `compilers <https://anaconda.org/conda-forge/compilers>`__
package, and its dependencies.
=====================
Installation prefices
=====================
Conda's environment configuration mean that dynamic linking is much
more tightly controlled than outside of an environment.
This means that, for example, building shared object libraries and
installing them into custom directories that you append to environment
variables such as ``LD_LIBRARY_PATH`` may not resolve links properly.
The way to avoid this is simply to always install code directly into
a conda environment.
For autotools builds this would just mean using ``--prefix=${CONDA_PREFIX}``
during ``./configure``.
==================
Compiling on macOS
==================
Compiling on macOS using conda compilers is not absolutely trivial, there is
a small set of steps that need to be followed to configure your environment.
.. _lscsoft-conda-compiling-macos-sdk:
~~~~~~~~~~~~~~~~~~~~~~~~
Installing the macOS SDK
~~~~~~~~~~~~~~~~~~~~~~~~
Conda compilers will try and compile software for the oldest compatible
version of macOS, which is 10.9 most of the time.
So, you will likely need to install the MacOSX10.9 SDK, which you can do
as follows:
.. code-block:: bash
mkdir -p /opt
cd /opt
wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz
tar -xf MacOSX10.9.sdk.tar.xz
Then, set the ``CONDA_BUILD_SYSROOT`` environment variable to match the new
SDK:
.. code-block:: bash
export CONDA_BUILD_SYSROOT="/opt/MacOSX10.9.sdk"
~~~~~~~~~~~~~~~~~~~~~~
Configuring the linker
~~~~~~~~~~~~~~~~~~~~~~
In order to resolve links, especially during ``./configure`` steps for
autotools builds, you will need to 'activate' your build environment with
the following steps:
.. code-block:: bash
conda activate <env-name>
export PREFIX="${CONDA_PREFIX}"
export CONDA_BUILD="1"
conda activate <env-name>
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config
from sphinx.util import logging
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'LSCSoft Conda'
copyright = '2019, The LIGO Scientific Collaboration'
author = 'The LIGO Scientific Collaboration'
# The short X.Y version
version = ''
# The full version, including alpha/beta/rc tags
release = ''
# -- General configuration ---------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinxcontrib.programoutput',
'sphinx.ext.intersphinx',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
"collapse_navigation": False,
"display_version": False,
"sticky_navigation": True,
"logo_only": True,
}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself. Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "_static/lsc-logo.png"
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'LSCSoftCondadoc'
# -- Options for LaTeX output ------------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'LSCSoftConda.tex', 'LSCSoft Conda Documentation',
'The LIGO Scientific Collaboration', 'manual'),
]
# -- Options for manual page output ------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'lscsoftconda', 'LSCSoft Conda Documentation',
[author], 1)
]
# -- Options for Texinfo output ----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'LSCSoftConda', 'LSCSoft Conda Documentation',
author, 'LSCSoftConda', 'One line description of project.',
'Miscellaneous'),
]
# -- Options for Epub output -------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = project
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''
# A unique identification for the text.
#
# epub_uid = ''
# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']
# -- Extensions ---------------------------------------------------------------
# Intersphinx
intersphinx_mapping = {
"python": ("https://docs.python.org/", None),
"conda": ("https://docs.conda.io/projects/conda/en/latest/", None),
"conda-build": ("https://docs.conda.io/projects/conda-build/en/latest/", None),
"conda-forge": ("https://conda-forge.org/docs/", None),
}
# -- Create extra content -----------------------------------------------------
def build_environment_list(_):
from pathlib import Path
from utils import (find_environments, write_environment)
logger = logging.getLogger('environments')
logger.info("[env] creating environment pages")
for env in find_environments(path=Path.cwd().parent):
envfn = Path('environments')/(env.stem + '.rst')
with open(envfn, 'w') as envf:
write_environment(env, file=envf)
logger.info("[env] created {0}".format(envfn))
# -- Setup --------------------------------------------------------------------
def setup(app):
app.connect('builder-inited', build_environment_list)
app.add_css_file("lscsoft-conda.css")
.. _lscsoft-conda-reference-environments:
======================
Reference environments
======================
The LSCSoft Conda team currently defines reference environments for a number
of versions of Python.
You can install these by downloading the relevant YAML file from the repository
and installing them.
.. _lscsoft-conda-pre-built-environments:
----------------------
Pre-built environments
----------------------
The reference environments are pre-built and distributed using
`CVMFS <https://cvmfs.readthedocs.io/>`_.
You can configure your system to use the distributed conda installation:
.. code-block:: bash
source /cvmfs/ligo-containers.opensciencegrid.org/lscsoft/conda/latest/etc/profile.d/conda.sh
You can then list the available environments:
.. code-block:: bash
conda env list
which will print something like:
.. program-output:: sh environments/list.sh
You can then activate one of these environments:
.. code-block:: bash
conda activate ligo-py37
.. _lscsoft-conda-available-environments:
----------------------
Available environments
----------------------
.. toctree::
:glob:
*
#!/bin/bash
CONDA_CMD="conda env list"
if [ -z "${LSCSOFT_CONDA_DOCKER_IMAGE}" ]; then
${CONDA_CMD}
else
docker run ${LSCSOFT_CONDA_DOCKER_IMAGE} ligo-py37 ${CONDA_CMD} | sed 's|/opt/conda|/cvmfs/ligo-containers.opensciencegrid.org/lscsoft/conda/latest|g'
fi
.. LSCSoft Conda documentation master file, created by
sphinx-quickstart on Wed Feb 6 08:39:38 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
.. sectionauthor:: Duncan Macleod <duncann.macleod@ligo.org>
.. title:: LSCSoft Conda
.. _lscsoft-conda:
.. include:: ../README.rst
.. toctree::
:caption: Documentation
:maxdepth: 2
usage/index
environments/index
compiling/index
packaging/index
tips/index
.. _lscsoft-conda-packaging:
*****************************
Packaging software with conda
*****************************
This document contains a brief primer on packaging software for distribution
with `conda <https://conda.io>`__.
.. _lscsoft-conda-packaging-recipe:
=================
What is a recipe?
=================
A recipe is a collection of files that define how to build a Conda package.
Each recipe minimally contains a ``meta.yaml`` file that describes the package
name and version, its dependencies, and how to build it.
-------------
``meta.yaml``
-------------
For a fairly exhaustive description of what you can put in the ``meta.yaml``
file, see :ref:`meta-yaml`.
------------
``build.sh``
------------
You can either define the build script inline in the ``meta.yaml`` file
(`example <https://github.com/conda-forge/gwpy-feedstock/blob/79c965de850fb69e621e0fecc5310ee50574d6fc/recipe/meta.yaml>`__),
otherwise ``conda-build`` will look for a ``build.sh`` script in the same
directory (``bld.bat`` for Windows).
The ``build.sh`` script is a shell script that can do whatever you need it to do
(`example <https://github.com/conda-forge/nds2-client-feedstock/blob/492436f4fa3c6b8a62dbc86dc68d79f818d251e3/recipe/build.sh>`__).
.. _lscsoft-conda-packaging-conda-forge:
=======================================
Packaging something new for conda-forge
=======================================
All new recipes should be submitted as a Pull Request to the
`conda-forge/staged-recipes <https://github.com/conda-forge/staged-recipes>`__
repository on GitHub.
The basic instructions are (from `here <https://conda-forge.org/#add_recipe>`__):
- create a new recipe in a sub-directory of the ``/recipes`` directory in ``staged-recipes``,
- commit this new recipe to a branch of your fork, and propose a Pull Request
- fix the inevitable linting and CI issues
Recipes can look quite different between languages, here are some good
examples from LSCSoft Conda (feel free to suggest your own):
- Python: `gwpy <https://github.com/conda-forge/gwpy-feedstock/tree/79c965de850fb69e621e0fecc5310ee50574d6fc/recipe/>`__,
- C/C++: `nds2-client <https://github.com/conda-forge/nds2-client-feedstock/tree/492436f4fa3c6b8a62dbc86dc68d79f818d251e3/recipe/>`__,
- Multiple outputs: `lal <https://github.com/conda-forge/lal-feedstock/tree/13e6fd9ba32f15b25aaf68bc6ffe106398d171b1/recipe>`__.
.. _lscsoft-conda-packaging-github:
==========================================
Maintaining a conda-forge recipe on GitHub
==========================================
Once a recipe has been reviewed and merged, a new feedstock repository
will be created to house the recipe, and all build and continuous
integration (CI) utilities required to automatically build and upload
packages.
All changes to the recipe from that point should be made via Pull Requests
against the feedstock repository.
There are automatic bots that will post Pull Requests whenever they think
that the recipe is out of date.
This is mainly done by searching for tarballs in the same location as the
current release tarball that suggest a newer version has been uploaded.
.. _lscsoft-conda-packaging-mkl:
===================================
Building packages against Intel MKL
===================================
.. warning::
This work is experimental, and has not been reviewed by anyone who knows
what they are doing. We are seeking advice from the conda-forge team.
All of the LSCSoft Conda packages are built using the conda-forge build
infrastructure, which defaults to using ``openblas`` builds of BLAS,
affecting GSL, numpy, scipy, and similar libraries.
It is possible to configure a conda package to build against MKL-builds of
BLAS, GSL, and friends; supporting both ``openblas`` and ``mkl`` builds at the
same time (switching using command-line options) requires some thought,
but isn't too hard.
If you have a recipe that is configured correctly, you can select an MKL
build from the command line using something like this:
.. code-block:: bash
conda build recipe --variants "{blas_impl: \"mkl\", mkl: \"2018\"}"
.. _lscsoft-conda-packaging-testing:
======================
Testing package builds
======================
------
Basics
------
In principle, testing a package build should be as simple as installing
``conda-build`` and then executing ``conda build``:
.. code-block:: bash
cd /path/to/recipe/parent/dir
conda build recipe/
e.g:
.. code-block:: bash
git clone https://github.com/conda-forge/lal-feedstock.git
cd lal-feedstock
conda build recipe
=============================
Common issues and workarounds
=============================
If your build has failed, try one of the following hacky workarounds.
If that doesn't work, try google.
----------------------------------------------------
(linux) C compiler doesn't see the ``USER`` variable
----------------------------------------------------
Add ``USER`` to the ``build/script-env`` list of variables to be inherited
into ``build.sh``:
.. code-block:: yaml
build:
script_env:
- USER
.. _lscsoft-conda-packaging-hacks-macos-sdk:
-------------------------------------------------------------
(macOS) Build tries to use the wrong version of the macOS SDK
-------------------------------------------------------------
Make sure and read :ref:`lscsoft-conda-compiling-macos-sdk`.
Then you can tell ``conda-build`` about this by adding the following
lines to ``~/conda_build_config.yaml`` (create it if you don't have it):
.. code-block:: yaml
CONDA_BUILD_SYSROOT: # [osx]
- /opt/MacOSX10.9.sdk # [osx]
=================
Docker containers
=================
If you prefer to work in a pristine environment, you can use the
`condaforge/linux-anvil
<https://hub.docker.com/r/condaforge/linux-anvil/>`__
docker container; this is what is used by conda-forge feedstock CIs
when building packages for production.
================================================================
Compiling packages in a Conda environment, not using conda-build
================================================================
This section has been moved to :ref:`lscsoft-conda-compiling`.
pyyaml
sphinx
sphinx_rtd_theme
sphinxcontrib-programoutput
tabulate
.. _lscsoft-conda-tips:
===============
Tips and tricks
===============
--------------------
Clone an environment
--------------------
You can clone one of the pre-built environments (or any environment, generally) with
.. code-block:: bash
conde create --name <target> --clone <source>
e.g.:
.. code-block:: bash
conda create --name myligo-py37 --clone ligo-py37
--------------------
Preserve your prompt
--------------------
By default, activating a conda environment will change your login prompt.
This behaviour can be disabled with
.. code-block:: bash
conda config --set changeps1 no
-----------
Using LaTeX
-----------
The LSCSoft Conda environments do not provide (La)TeX.
You should install TeX using your system package manager, see below for
details for some common operating systems:
.. note::
The following examples should get you far enough to use TeX with
the Matplotlib Python library. If you find that other packages
are needed, please
`open a Bug ticket <https://git.ligo.org/lscsoft/conda/issues/new?issuable_template=Bug>`_.
^^^^^^^^
Macports
^^^^^^^^
.. code-block:: bash
port install texlive dvipng
.. _lscsoft-conda-usage:
===================
Using LSCSoft Conda
===================
.. note::
LSCSoft Conda is fully supported on 64-bit Linux and macOS, support for
other platforms is on a best-effort basis only.
.. note::
Linux users, especially those using the LIGO Data Grid computing centres,
may find it easier to just use one of the
:ref:`lscsoft-conda-pre-built-environments`.
----------------
1. Install conda
----------------
Using software provided by LSCSoft Conda requires the
`conda <https://conda.io>`_ package manager.
We recommend `installing Miniconda
<https://conda.io/projects/conda/en/latest/user-guide/install/index.html>`_,
however you can also `install conda using your Linux package manager
<https://www.anaconda.com/rpm-and-debian-repositories-for-miniconda/>`_.
-------------------------------
2. Add the conda-forge channnel
-------------------------------
All LSCSoft Conda packages are distributed on the conda-forge channnel.
.. code-block:: shell
conda config --add channels conda-forge
------------------------
3. Create an environment
------------------------
You should now be able to create a conda environment:
.. code-block:: bash
conda create --name lscsoft python=3.7 gwpy pycbc
#!/usr/bin/env python
"""Format the current list of environments for inclusion in the sphinx docs
"""
import json
import os.path
import pathlib
import re
from subprocess import check_output
import yaml
from sphinx.util import logging
from tabulate import tabulate
logger = logging.getLogger('utils')
ANACONDA_CHANNELS = {
"pkgs/free",
"pkgs/main",
}
DOCKER_IMAGE = os.getenv("LSCSOFT_CONDA_DOCKER_IMAGE", None)
def run_conda_command(cmd):
if DOCKER_IMAGE:
cmd = ["docker", "run", DOCKER_IMAGE, "ligo-py37"] + cmd
logger.info("[utils] $ {0}".format(" ".join(cmd)))
return check_output(cmd)
def find_environments(path=None):
"""Finds all of the environments known to conda
"""
# get files
if path is None:
path = pathlib.Path(__file__).resolve().parent.parent
files = path.glob('environment-*.yml')
# get environments
envs = json.loads(run_conda_command([
'conda',
'env',
'list',
'--json',
'--quiet',
]).strip().decode())['envs']
env_names = [pathlib.Path(env).name for env in envs]
# cross-check
for fn in files:
with fn.open("rb") as envf:
content = yaml.load(envf)
name = content['name']
if name in env_names:
yield fn
def write_environment(env, file=None):
env = pathlib.Path(env)
with env.open("rb") as envf:
content = yaml.load(envf)
# write name
name = content['name']
print("*" * len(name), file=file)
print(name, file=file)
print("*" * len(name), file=file)
print("", file=file)
# write link
print("| **Download:** `{0} <https://git.ligo.org/lscsoft/conda/raw/master/"
"{0}>`_".format(env.name),
file=file,
)
channels = content['channels']
if 'nodefaults' not in channels:
channels.insert(0, 'defaults')
print("| **Channels:** ``{0}``".format("``, ``".join(channels)),
file=file)
print("| **Install:**\n\n::\n\n"
" conda env create --file {0}\n".format(env.name),
file=file)
# get packages
celist = run_conda_command([
'conda',
'list',
'--name',
name,
'--json',
'--quiet',
])
packages = json.loads(celist)
# write packages
print("| **Packages:**\n", file=file)
rows = []
images = {}
for pkg in packages:
cloudchannel = ('anaconda' if pkg['channel'] in ANACONDA_CHANNELS else
pkg['channel'])
rows.append((
"`{name} <https://anaconda.org/{cloudchannel}/{name}/>`_".format(
cloudchannel=cloudchannel,
**pkg
),
pkg['version'],
pkg['build_string'],
pkg['channel'],
))
print(
tabulate(
rows,
headers=('Name', 'Version', 'Build', 'Channel'),
tablefmt="rst",
),
file=file,
)
#!/bin/bash
# mimic a login shell by sourcing files under /etc/profile.d
# without using "-l" argument to bash a full login shell really
# wants a TTY that docker doesn't provide by default
. /etc/profile.d/conda.sh
conda activate $1
shift
exec "$@"
name: lal-development
channels:
- conda-forge/label/gcc7
- conda-forge
dependencies:
- python=3.6
- nomkl
- swig >= 3.0.9
- pkg-config
- autoconf
- automake
- bc
- cfitsio
- chealpix
- clang_osx-64
- gsl
- libframe
- zlib
- openblas
- doxygen
- fftw
- gsl
- h5py
- hdf5
- healpy
- ldas-tools-framecpp
- libframe
- ligo-segments
- lscsoft-glue
- make
- metaio
- numpy
- pkg-config
- pytest
- python = 3.6
- scipy
- six
- swig >= 3.0.12
- texlive-core
- zlib
name: lal-development
channels:
- conda-forge/label/gcc7
- conda-forge
dependencies:
- python=3.6
- nomkl
- swig >= 3.0.9
- pkg-config
- autoconf
- automake
- bc
- cfitsio
- chealpix
- doxygen
- fftw
- gcc_linux-64
- gsl
- zlib
- openblas
- libframe
- fftw
- h5py
- hdf5
- healpy
- ldas-tools-framecpp
- libframe
- ligo-segments
- lscsoft-glue
- make
- metaio
- numpy
- pkg-config
- pytest
- python = 3.6
- scipy
- six
- swig >= 3.0.12
- texlive-core
- zlib
......@@ -3,39 +3,57 @@ channels:
- conda-forge
dependencies:
- python=2.7
# common build tools
- autoconf
- automake
- bc
- blas
- compilers
- cmake
- gsl
- pkg-config
- swig
- wget
# python tools
- flake8
- ipython
- jupyter
- pyflakes
- pylint
# lscsoft packages
- bayeswave
- bayeswaveutils
- bilby
- bruco
- dqsegdb
- dqsegdb2
- gpstime
- gwdatafind
- gwdetchar
- gwosc
- gwpy
- hveto
- kombine
- lal
- lalburst
- lalframe
- lalinspiral
- lalmetaio
- lalpulsar
- lalsimulation
- lalxml
- lalsuite
- lal=*=mkl*
- ldas-tools-al
- ldas-tools-cmake
- ldas-tools-diskcacheapi
- ldas-tools-filters
- ldas-tools-frameapi
- ldas-tools-framecpp
- ldas-tools-ldasgen
- libframe
- ligo-common
- ligo-gracedb
- ligo-lvalert
- ligo-segments
- ligotimegps
- lscsoft-glue
- metaio
- nds2-client
- python-lal
- python-lalburst
- python-lalframe
- python-lalinspiral
- python-lalmetaio
- python-lalpulsar
- python-lalsimulation
- python-lalxml
- pycbc
- python-ldas-tools-al
- python-ldas-tools-framecpp
- python-ldas-tools-ldasgen
- python-ligo-lw
- python-nds2-client
- virtualenv
name: ligo-py35
channels:
- conda-forge
dependencies:
- python=3.5
- dqsegdb2
- gpstime
- gwdatafind
- gwosc
- gwpy
- kombine
- lal
# - lalburst
- lalframe