Commit 7afc4e8e authored by Jonathan Hanks's avatar Jonathan Hanks

Merge branch 'system_tests' into 'master'

System tests

See merge request !2
parents e0ad574e 2ee249ce
Pipeline #45960 passed with stage
in 9 minutes and 49 seconds
#------------------------------------------------------------------------
# gitlab-runner exec docker --timeout 7200 pages --docker-volumes ${HOME}/Sources/GDS/nds/nds2-test-blobs:/replay_data <target>
variables:
DOCKER_DRIVER: overlay
BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
NIGHTLY: $CI_REGISTRY_IMAGE:nightly
TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
stages:
- build-and-test
before_script:
- ulimit -S -c 0
#========================================================================
# A N C H O R S
#========================================================================
#------------------------------------------------------------------------
# images
#------------------------------------------------------------------------
.template-image-deb-buster: &image-deb-buster
image: debian:buster
.template-image-deb-stretch: &image-deb-stretch
image: ligo/base:stretch
.template-image-deb-jessie: &image-deb-jessie
image: ligo/base:jessie
#------------------------------------------------------------------------
# Debian
#------------------------------------------------------------------------
.deb:build: &deb-build-and-test
script:
#--------------------------------------------------------------------
# First update package list and then ...
# Extract the build dependencies and get them installed
#--------------------------------------------------------------------
- apt-get update
- apt-get install -y build-essential cmake libboost-dev libpstreams-dev
- rm -rf ${CI_PROJECT_DIR}/cmake-build
- mkdir ${CI_PROJECT_DIR}/cmake-build
- cd ${CI_PROJECT_DIR}/cmake-build
- cmake -DCMAKE_INSTALL_PREFIX=/usr ..
- cmake --build . -- VERBOSE=1
- ctest
- DESTDIR=${CI_PROJECT_DIR}/cmake-build/t cmake --build . --target install
artifacts:
expire_in: 1h
paths:
- cmake-build/t
only:
- pushes
- schedules
ligo-channel-completion:buster:
stage: build-and-test
<<: *image-deb-buster
<<: *deb-build-and-test
ligo-channel-completion:stretch:
stage: build-and-test
<<: *image-deb-stretch
<<: *deb-build-and-test
ligo-channel-completion:jessie:
stage: build-and-test
<<: *image-deb-jessie
<<: *deb-build-and-test
\ No newline at end of file
A bash (and soon other) completion for LIGO channel names. A bash (and soon other) completion for LIGO channel names.
This provides a bash completion script along with a channel completion program. This provides a bash completion script along with a channel completion program.
A channel database is required. The database is a simple flat text file where A set of channel databases are required. Each database is a simple flat text file where
every line is a single channel name. The file must be sorted. The path to the every line is a single channel name. The file must be sorted.
file is set via the 'CHAN_LIST' environment variable.
If you have a list of channels you can have the application sort it for you The path to the files is set via the 'EPICS_CHAN_LIST' and 'NDS_CHAN_LIST' environment variables.
As the names imply the database are for EPICS channels or DAQ/NDS channels. This split is needed
as in the LIGO control room there are > 500k channels, with roughly half are available only as EPICS channels.
By specifically supporting EPICS channels then non-daq channels like string records can be completed against
for caget/caput/...
If you have a list of channels you can have the application sort it for you. Please note that
it is the bash scripts that use the EPICS_CHAN_LIST/NDS_CHAN_LIST variables, the completion engine
directly takes the database on the command line.
<pre> <pre>
# assuming that the channel database is # assuming that the channel database is
# located at a path specified by the # located at db.txt.
# CHAN_LIST environment variable ligo_channel_completion -d db.txt -r > tmpfile
channel_completion -r > tmpfile mv tmpfile db.txt
mv tmpfile "$CHAN_LIST"
</pre> </pre>
Requirements: Requirements:
* A C++11 compliant compiler. * A C++11 compliant compiler.
* CMake 3+ * CMake 3+
* Boost iterator library
* pstreams library
Debian requirements: Debian requirements:
* build-essential * build-essential
* bash-completion * bash-completion
* libboost-dev
* libpstreams-dev
Tested on: Tested on:
* Debian 8 & 9 * Debian 8 & 9
...@@ -43,9 +54,11 @@ make install ...@@ -43,9 +54,11 @@ make install
Components installed (Assuming a install prefix of /usr): Components installed (Assuming a install prefix of /usr):
* a binary ligo_channel_completion to /usr/lib * a binary ligo_channel_completion to /usr/lib
* a shell script to the base completion libraries at /usr/share/bash-completion/completions * a set shell script to the base completion libraries at /usr/share/bash-completion/completions
After the install is successful, start a new bash shell and use the channel completion. After the install is successful, start a new bash shell and use the channel completion and make
sure that the EPICS_CHAN_LIST and NDS_CHAN_LIST environment variables are exported and refer
to proper databases.
<pre> <pre>
$ ndscope H&lt;tab&gt; $ ndscope H&lt;tab&gt;
......
# Add commands that can use LIGO channel completion to this list # Add commands that can use LIGO channel completion to this list
set (COMPLETE_THESE_COMMANDS set (COMPLETE_THESE_NDS_COMMANDS
ndscope ndscope
)
set (COMPLETE_THESE_EPICS_COMMANDS
cdsutils cdsutils
caget caget
caput caput
camonitor camonitor
probe
) )
set(COMPLETE_COMMAND_REGISTRATION) set(COMPLETE_NDS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_COMMANDS}) foreach(entry ${COMPLETE_THESE_NDS_COMMANDS})
set(COMPLETE_COMMAND_REGISTRATION "${COMPLETE_COMMAND_REGISTRATION} set(COMPLETE_NDS_COMMAND_REGISTRATION "${COMPLETE_NDS_COMMAND_REGISTRATION}
complete -F _channel_completion ${entry}") complete -F _ligo_nds_channel_completion ${entry}")
endforeach()
set(COMPLETE_EPICS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_EPICS_COMMANDS})
set(COMPLETE_EPICS_COMMAND_REGISTRATION "${COMPLETE_EPICS_COMMAND_REGISTRATION}
complete -F _ligo_epics_channel_completion ${entry}")
endforeach() endforeach()
set(COMPLETE_SCRIPT_DIR "${CMAKE_INSTALL_PREFIX}/share/bash-completion/completions") set(COMPLETE_SCRIPT_DIR "${CMAKE_INSTALL_PREFIX}/share/bash-completion/completions")
configure_file(ligo_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion @ONLY) configure_file(ligo_nds_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion @ONLY)
configure_file(ligo_epics_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion @ONLY)
configure_file(link_file.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake @ONLY) configure_file(link_file.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion" install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR})
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR}) DESTINATION ${COMPLETE_SCRIPT_DIR})
install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake) install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake)
# Copyright 2019 California Institute of Technology.
# You should have received a copy of the licensing terms for this
# software included in the file “LICENSE” located in the top-level
# directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt
_ligo_epics_channel_completion()
{
local cur
cur="${COMP_WORDS[COMP_CWORD]}"
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${EPICS_CHAN_LIST}") )
__ltrim_colon_completions "$cur"
return 0
}
@COMPLETE_EPICS_COMMAND_REGISTRATION@
\ No newline at end of file
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
# directory of this package. If you did not, you can view a copy at # directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt # http://dcc.ligo.org/M1500244/LICENSE.txt
_channel_completion() _ligo_nds_channel_completion()
{ {
local cur local cur
cur="${COMP_WORDS[COMP_CWORD]}" cur="${COMP_WORDS[COMP_CWORD]}"
...@@ -14,10 +14,10 @@ _channel_completion() ...@@ -14,10 +14,10 @@ _channel_completion()
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480 # see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur _get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}") ) COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${NDS_CHAN_LIST}") )
__ltrim_colon_completions "$cur" __ltrim_colon_completions "$cur"
return 0 return 0
} }
@COMPLETE_COMMAND_REGISTRATION@ @COMPLETE_NDS_COMMAND_REGISTRATION@
\ No newline at end of file \ No newline at end of file
...@@ -5,10 +5,19 @@ ...@@ -5,10 +5,19 @@
# variable (if present) which is found at run/install time, not at configuration time. # variable (if present) which is found at run/install time, not at configuration time.
# ideas from https://stackoverflow.com/questions/35765106/symbolic-links-cmake # ideas from https://stackoverflow.com/questions/35765106/symbolic-links-cmake
set (LINK_ENTRIES "@COMPLETE_THESE_COMMANDS@")
set (LINK_ENTRIES "@COMPLETE_THESE_NDS_COMMANDS@")
foreach (entry ${LINK_ENTRIES}) foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_channel_completion ${entry} execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_nds_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@" WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
) )
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@") message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach() endforeach()
set (LINK_ENTRIES "@COMPLETE_THESE_EPICS_COMMANDS@")
foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_epics_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
)
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach()
\ No newline at end of file
find_package(Boost REQUIRED)
add_library(completion INTERFACE) add_library(completion INTERFACE)
target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} ${Boost_INCLUDEDIR})
target_requires_cpp11(completion INTERFACE) target_requires_cpp11(completion INTERFACE)
add_executable(test_completion_lib add_executable(test_completion_lib
......
...@@ -11,32 +11,67 @@ ...@@ -11,32 +11,67 @@
#include <algorithm> #include <algorithm>
#include <iterator> #include <iterator>
#include <fstream> #include <fstream>
#include <ostream>
#include <string> #include <string>
#include <vector> #include <vector>
#include <cstring> #include <cstring>
#include <boost/iterator/transform_iterator.hpp>
namespace completion namespace completion
{ {
typedef std::vector< std::string > string_list; typedef std::vector< std::string > string_list;
struct channel
{
std::string name;
std::string extra;
channel( ) = default;
channel( std::string name, std::string extra = "" )
: name( std::move( name ) ), extra( std::move( extra ) )
{
}
};
std::ostream&
operator<<( std::ostream& os, const channel& ch )
{
os << ch.name;
if ( !ch.extra.empty( ) )
{
os << " " << ch.extra;
}
return os;
}
struct Database struct Database
{ {
std::vector< std::string > channels; std::vector< channel > channels;
}; };
namespace detail
{
const std::string&
project_name( const channel& entry )
{
return entry.name;
}
}
inline bool inline bool
simple_comparison( const std::string& a, const std::string& b ) simple_comparison( const channel& a, const channel& b )
{ {
return std::strcmp( a.c_str( ), b.c_str( ) ) < 0; return std::strcmp( a.name.c_str( ), b.name.c_str( ) ) < 0;
} }
inline bool inline bool
key_comparison( const std::string& key, const std::string& element ) key_comparison( const std::string& key, const channel& element )
{ {
int result = int result =
std::strncmp( key.c_str( ), element.c_str( ), key.size( ) ); std::strncmp( key.c_str( ), element.name.c_str( ), key.size( ) );
return result < 0; return result < 0;
} }
...@@ -48,17 +83,28 @@ namespace completion ...@@ -48,17 +83,28 @@ namespace completion
std::string line; std::string line;
while ( std::getline( input, line ) ) while ( std::getline( input, line ) )
{ {
db.channels.emplace_back( line ); auto space = line.find( ' ' );
std::string extra;
if ( space < std::string::npos )
{
extra = line.substr( space + 1 );
line.resize( space );
}
db.channels.emplace_back( line, extra );
} }
return db; return db;
} }
template < typename It > template < typename It >
string_list string_list
search( It begin_it, It end_it, const std::string& key ) search( It begin_it_, It end_it_, const std::string& key )
{ {
string_list results; string_list results;
auto begin_it =
boost::make_transform_iterator( begin_it_, detail::project_name );
auto end_it =
boost::make_transform_iterator( end_it_, detail::project_name );
auto lower = std::lower_bound( begin_it, end_it, key, key_comparison ); auto lower = std::lower_bound( begin_it, end_it, key, key_comparison );
auto upper = std::upper_bound( lower, end_it, key, key_comparison ); auto upper = std::upper_bound( lower, end_it, key, key_comparison );
auto key_size = key.size( ); auto key_size = key.size( );
......
...@@ -5,16 +5,187 @@ ...@@ -5,16 +5,187 @@
// directory of this package. If you did not, you can view a copy at // directory of this package. If you did not, you can view a copy at
// http://dcc.ligo.org/M1500244/LICENSE.txt // http://dcc.ligo.org/M1500244/LICENSE.txt
#include <sstream>
#include "completion.hh" #include "completion.hh"
#include "catch.hpp" #include "catch.hpp"
class temporary_fd
{
public:
explicit temporary_fd( std::string temp_dir ) : fd_( -1 ), name_( )
{
std::string path( std::move( temp_dir ) );
if ( !path.empty( ) )
{
if ( path.back( ) != '/' )
{
path.push_back( '/' );
}
}
path += "tmpfileXXXXXX";
std::vector< char > tmp;
tmp.reserve( path.size( ) + 1 );
std::copy( path.begin( ), path.end( ), std::back_inserter( tmp ) );
tmp.push_back( '\0' );
name_.clear( );
name_.reserve( tmp.size( ) );
fd_ = mkstemp( tmp.data( ) );
if ( fd_ >= 0 )
{
name_.append( tmp.data( ) );
}
}
temporary_fd( const temporary_fd& other ) = delete;
temporary_fd( temporary_fd&& other ) noexcept
: fd_( other.fd_ ),
name_( std::move( other.name_ ) )
{
other.fd_ = -1;
}
temporary_fd operator=( const temporary_fd& other ) = delete;
temporary_fd&
operator=( temporary_fd&& other ) noexcept
{
unlink( );
fd_ = other.fd_;
name_ = std::move( other.name_ );
other.fd_ = -1;
other.name_.clear( );
return *this;
};
~temporary_fd( )
{
unlink( );
}
void
add_data( const std::string& data )
{
if ( *this )
{
size_t remaining = data.size( );
size_t copied = 0;
while ( remaining )
{
ssize_t count = write( fd_, data.data( ) + copied, remaining );
if ( count >= 0 )
{
remaining -= static_cast< size_t >( count );
copied += static_cast< size_t >( count );
}
else
{
auto err = errno;
if ( err == EAGAIN || err == EINTR )
{
continue;
}
throw std::runtime_error(
"Error writing data to temp file" );
}
}
}
}
void
add_data( const std::vector< std::string >& data )
{
for ( const auto& cur : data )
{
add_data( cur );
add_data( "\n" );
}
}
void
close( )
{
if ( fd_ >= 0 )
{
::close( fd_ );
}
fd_ = -1;
}
std::string
filename( ) const
{
return name_;
}
operator bool( ) const
{
return fd_ >= 0;
}
private:
void
unlink( )
{
close( );
if ( !name_.empty( ) )
{
::unlink( name_.c_str( ) );
}
name_.clear( );
}
int fd_;
std::string name_;
};
TEST_CASE( "Load empty database" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{} );
auto db = completion::load_database( file.filename( ) );
REQUIRE( db.channels.size( ) == 0 );
}
TEST_CASE( "Load database with no spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{ "abc", "def" } );
auto db = completion::load_database( file.filename( ) );
auto expected =
std::vector< completion::channel >{ { "abc", "" }, { "def", "" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Load database with spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{
"abc", "def", "ghi jkl", "mno ", "pqr stu" } );
auto db = completion::load_database( file.filename( ) );
auto expected = std::vector< completion::channel >{ { "abc", "" },
{ "def", "" },
{ "ghi", "jkl" },
{ "mno", "" },
{ "pqr", "stu" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Basic completion behavior" ) TEST_CASE( "Basic completion behavior" )
{ {
completion::Database db; completion::Database db;
db.channels = { db.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3", { "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C", { "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
}; };
{ {
...@@ -53,19 +224,35 @@ TEST_CASE( "Basic DB sorting" ) ...@@ -53,19 +224,35 @@ TEST_CASE( "Basic DB sorting" )
{ {
completion::Database input; completion::Database input;
input.channels = { input.channels = {
"H0:VAC-SYSTEM_3", "H0:VAC-SYSTEM_1", "H1:SYS-SUB_C", { "H0:VAC-SYSTEM_3" }, { "H0:VAC-SYSTEM_1" }, { "H1:SYS-SUB_C" },
"H0:VAC-SYSTEM_2", "H1:SYS-SUB_A", "H1:SYS-SUB_B", { "H0:VAC-SYSTEM_2" }, { "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" },
}; };
completion::Database expected; completion::Database expected;
expected.channels = { expected.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3", { "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C", { "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
}; };
completion::sort( input ); completion::sort( input );
REQUIRE( input.channels.size( ) == expected.channels.size( ) ); REQUIRE( input.channels.size( ) == expected.channels.size( ) );
for ( int i = 0; i < input.channels.size( ); ++i ) for ( int i = 0; i < input.channels.size( ); ++i )
{ {
REQUIRE( input.channels[ i ] == expected.channels[ i ] ); REQUIRE( input.channels[ i ].name == expected.channels[ i ].name );
} }
}
TEST_CASE( "Serialization of channel with an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc", "def" );
os << a;
REQUIRE( os.str( ) == "abc def" );
}
TEST_CASE( "Serialization of channel without an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc" );
os << a;