Commit 7afc4e8e authored by Jonathan Hanks's avatar Jonathan Hanks

Merge branch 'system_tests' into 'master'

System tests

See merge request !2
parents e0ad574e 2ee249ce
Pipeline #45960 passed with stage
in 9 minutes and 49 seconds
#------------------------------------------------------------------------
# gitlab-runner exec docker --timeout 7200 pages --docker-volumes ${HOME}/Sources/GDS/nds/nds2-test-blobs:/replay_data <target>
variables:
DOCKER_DRIVER: overlay
BRANCH: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
COMMIT: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
NIGHTLY: $CI_REGISTRY_IMAGE:nightly
TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
stages:
- build-and-test
before_script:
- ulimit -S -c 0
#========================================================================
# A N C H O R S
#========================================================================
#------------------------------------------------------------------------
# images
#------------------------------------------------------------------------
.template-image-deb-buster: &image-deb-buster
image: debian:buster
.template-image-deb-stretch: &image-deb-stretch
image: ligo/base:stretch
.template-image-deb-jessie: &image-deb-jessie
image: ligo/base:jessie
#------------------------------------------------------------------------
# Debian
#------------------------------------------------------------------------
.deb:build: &deb-build-and-test
script:
#--------------------------------------------------------------------
# First update package list and then ...
# Extract the build dependencies and get them installed
#--------------------------------------------------------------------
- apt-get update
- apt-get install -y build-essential cmake libboost-dev libpstreams-dev
- rm -rf ${CI_PROJECT_DIR}/cmake-build
- mkdir ${CI_PROJECT_DIR}/cmake-build
- cd ${CI_PROJECT_DIR}/cmake-build
- cmake -DCMAKE_INSTALL_PREFIX=/usr ..
- cmake --build . -- VERBOSE=1
- ctest
- DESTDIR=${CI_PROJECT_DIR}/cmake-build/t cmake --build . --target install
artifacts:
expire_in: 1h
paths:
- cmake-build/t
only:
- pushes
- schedules
ligo-channel-completion:buster:
stage: build-and-test
<<: *image-deb-buster
<<: *deb-build-and-test
ligo-channel-completion:stretch:
stage: build-and-test
<<: *image-deb-stretch
<<: *deb-build-and-test
ligo-channel-completion:jessie:
stage: build-and-test
<<: *image-deb-jessie
<<: *deb-build-and-test
\ No newline at end of file
A bash (and soon other) completion for LIGO channel names.
This provides a bash completion script along with a channel completion program.
A channel database is required. The database is a simple flat text file where
every line is a single channel name. The file must be sorted. The path to the
file is set via the 'CHAN_LIST' environment variable.
A set of channel databases are required. Each database is a simple flat text file where
every line is a single channel name. The file must be sorted.
If you have a list of channels you can have the application sort it for you
The path to the files is set via the 'EPICS_CHAN_LIST' and 'NDS_CHAN_LIST' environment variables.
As the names imply the database are for EPICS channels or DAQ/NDS channels. This split is needed
as in the LIGO control room there are > 500k channels, with roughly half are available only as EPICS channels.
By specifically supporting EPICS channels then non-daq channels like string records can be completed against
for caget/caput/...
If you have a list of channels you can have the application sort it for you. Please note that
it is the bash scripts that use the EPICS_CHAN_LIST/NDS_CHAN_LIST variables, the completion engine
directly takes the database on the command line.
<pre>
# assuming that the channel database is
# located at a path specified by the
# CHAN_LIST environment variable
channel_completion -r > tmpfile
mv tmpfile "$CHAN_LIST"
# located at db.txt.
ligo_channel_completion -d db.txt -r > tmpfile
mv tmpfile db.txt
</pre>
Requirements:
* A C++11 compliant compiler.
* CMake 3+
* Boost iterator library
* pstreams library
Debian requirements:
* build-essential
* bash-completion
* libboost-dev
* libpstreams-dev
Tested on:
* Debian 8 & 9
......@@ -43,9 +54,11 @@ make install
Components installed (Assuming a install prefix of /usr):
* a binary ligo_channel_completion to /usr/lib
* a shell script to the base completion libraries at /usr/share/bash-completion/completions
* a set shell script to the base completion libraries at /usr/share/bash-completion/completions
After the install is successful, start a new bash shell and use the channel completion.
After the install is successful, start a new bash shell and use the channel completion and make
sure that the EPICS_CHAN_LIST and NDS_CHAN_LIST environment variables are exported and refer
to proper databases.
<pre>
$ ndscope H&lt;tab&gt;
......
# Add commands that can use LIGO channel completion to this list
set (COMPLETE_THESE_COMMANDS
set (COMPLETE_THESE_NDS_COMMANDS
ndscope
)
set (COMPLETE_THESE_EPICS_COMMANDS
cdsutils
caget
caput
camonitor
probe
)
set(COMPLETE_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_COMMANDS})
set(COMPLETE_COMMAND_REGISTRATION "${COMPLETE_COMMAND_REGISTRATION}
complete -F _channel_completion ${entry}")
set(COMPLETE_NDS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_NDS_COMMANDS})
set(COMPLETE_NDS_COMMAND_REGISTRATION "${COMPLETE_NDS_COMMAND_REGISTRATION}
complete -F _ligo_nds_channel_completion ${entry}")
endforeach()
set(COMPLETE_EPICS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_EPICS_COMMANDS})
set(COMPLETE_EPICS_COMMAND_REGISTRATION "${COMPLETE_EPICS_COMMAND_REGISTRATION}
complete -F _ligo_epics_channel_completion ${entry}")
endforeach()
set(COMPLETE_SCRIPT_DIR "${CMAKE_INSTALL_PREFIX}/share/bash-completion/completions")
configure_file(ligo_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion @ONLY)
configure_file(ligo_nds_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion @ONLY)
configure_file(ligo_epics_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion @ONLY)
configure_file(link_file.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion"
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR})
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR})
install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake)
# Copyright 2019 California Institute of Technology.
# You should have received a copy of the licensing terms for this
# software included in the file “LICENSE” located in the top-level
# directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt
_ligo_epics_channel_completion()
{
local cur
cur="${COMP_WORDS[COMP_CWORD]}"
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${EPICS_CHAN_LIST}") )
__ltrim_colon_completions "$cur"
return 0
}
@COMPLETE_EPICS_COMMAND_REGISTRATION@
\ No newline at end of file
......@@ -6,7 +6,7 @@
# directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt
_channel_completion()
_ligo_nds_channel_completion()
{
local cur
cur="${COMP_WORDS[COMP_CWORD]}"
......@@ -14,10 +14,10 @@ _channel_completion()
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}") )
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${NDS_CHAN_LIST}") )
__ltrim_colon_completions "$cur"
return 0
}
@COMPLETE_COMMAND_REGISTRATION@
\ No newline at end of file
@COMPLETE_NDS_COMMAND_REGISTRATION@
\ No newline at end of file
......@@ -5,10 +5,19 @@
# variable (if present) which is found at run/install time, not at configuration time.
# ideas from https://stackoverflow.com/questions/35765106/symbolic-links-cmake
set (LINK_ENTRIES "@COMPLETE_THESE_COMMANDS@")
set (LINK_ENTRIES "@COMPLETE_THESE_NDS_COMMANDS@")
foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_channel_completion ${entry}
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_nds_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
)
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach()
set (LINK_ENTRIES "@COMPLETE_THESE_EPICS_COMMANDS@")
foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_epics_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
)
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach()
\ No newline at end of file
find_package(Boost REQUIRED)
add_library(completion INTERFACE)
target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} ${Boost_INCLUDEDIR})
target_requires_cpp11(completion INTERFACE)
add_executable(test_completion_lib
......
......@@ -11,32 +11,67 @@
#include <algorithm>
#include <iterator>
#include <fstream>
#include <ostream>
#include <string>
#include <vector>
#include <cstring>
#include <boost/iterator/transform_iterator.hpp>
namespace completion
{
typedef std::vector< std::string > string_list;
struct channel
{
std::string name;
std::string extra;
channel( ) = default;
channel( std::string name, std::string extra = "" )
: name( std::move( name ) ), extra( std::move( extra ) )
{
}
};
std::ostream&
operator<<( std::ostream& os, const channel& ch )
{
os << ch.name;
if ( !ch.extra.empty( ) )
{
os << " " << ch.extra;
}
return os;
}
struct Database
{
std::vector< std::string > channels;
std::vector< channel > channels;
};
namespace detail
{
const std::string&
project_name( const channel& entry )
{
return entry.name;
}
}
inline bool
simple_comparison( const std::string& a, const std::string& b )
simple_comparison( const channel& a, const channel& b )
{
return std::strcmp( a.c_str( ), b.c_str( ) ) < 0;
return std::strcmp( a.name.c_str( ), b.name.c_str( ) ) < 0;
}
inline bool
key_comparison( const std::string& key, const std::string& element )
key_comparison( const std::string& key, const channel& element )
{
int result =
std::strncmp( key.c_str( ), element.c_str( ), key.size( ) );
std::strncmp( key.c_str( ), element.name.c_str( ), key.size( ) );
return result < 0;
}
......@@ -48,17 +83,28 @@ namespace completion
std::string line;
while ( std::getline( input, line ) )
{
db.channels.emplace_back( line );
auto space = line.find( ' ' );
std::string extra;
if ( space < std::string::npos )
{
extra = line.substr( space + 1 );
line.resize( space );
}
db.channels.emplace_back( line, extra );
}
return db;
}
template < typename It >
string_list
search( It begin_it, It end_it, const std::string& key )
search( It begin_it_, It end_it_, const std::string& key )
{
string_list results;
auto begin_it =
boost::make_transform_iterator( begin_it_, detail::project_name );
auto end_it =
boost::make_transform_iterator( end_it_, detail::project_name );
auto lower = std::lower_bound( begin_it, end_it, key, key_comparison );
auto upper = std::upper_bound( lower, end_it, key, key_comparison );
auto key_size = key.size( );
......
......@@ -5,16 +5,187 @@
// directory of this package. If you did not, you can view a copy at
// http://dcc.ligo.org/M1500244/LICENSE.txt
#include <sstream>
#include "completion.hh"
#include "catch.hpp"
class temporary_fd
{
public:
explicit temporary_fd( std::string temp_dir ) : fd_( -1 ), name_( )
{
std::string path( std::move( temp_dir ) );
if ( !path.empty( ) )
{
if ( path.back( ) != '/' )
{
path.push_back( '/' );
}
}
path += "tmpfileXXXXXX";
std::vector< char > tmp;
tmp.reserve( path.size( ) + 1 );
std::copy( path.begin( ), path.end( ), std::back_inserter( tmp ) );
tmp.push_back( '\0' );
name_.clear( );
name_.reserve( tmp.size( ) );
fd_ = mkstemp( tmp.data( ) );
if ( fd_ >= 0 )
{
name_.append( tmp.data( ) );
}
}
temporary_fd( const temporary_fd& other ) = delete;
temporary_fd( temporary_fd&& other ) noexcept
: fd_( other.fd_ ),
name_( std::move( other.name_ ) )
{
other.fd_ = -1;
}
temporary_fd operator=( const temporary_fd& other ) = delete;
temporary_fd&
operator=( temporary_fd&& other ) noexcept
{
unlink( );
fd_ = other.fd_;
name_ = std::move( other.name_ );
other.fd_ = -1;
other.name_.clear( );
return *this;
};
~temporary_fd( )
{
unlink( );
}
void
add_data( const std::string& data )
{
if ( *this )
{
size_t remaining = data.size( );
size_t copied = 0;
while ( remaining )
{
ssize_t count = write( fd_, data.data( ) + copied, remaining );
if ( count >= 0 )
{
remaining -= static_cast< size_t >( count );
copied += static_cast< size_t >( count );
}
else
{
auto err = errno;
if ( err == EAGAIN || err == EINTR )
{
continue;
}
throw std::runtime_error(
"Error writing data to temp file" );
}
}
}
}
void
add_data( const std::vector< std::string >& data )
{
for ( const auto& cur : data )
{
add_data( cur );
add_data( "\n" );
}
}
void
close( )
{
if ( fd_ >= 0 )
{
::close( fd_ );
}
fd_ = -1;
}
std::string
filename( ) const
{
return name_;
}
operator bool( ) const
{
return fd_ >= 0;
}
private:
void
unlink( )
{
close( );
if ( !name_.empty( ) )
{
::unlink( name_.c_str( ) );
}
name_.clear( );
}
int fd_;
std::string name_;
};
TEST_CASE( "Load empty database" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{} );
auto db = completion::load_database( file.filename( ) );
REQUIRE( db.channels.size( ) == 0 );
}
TEST_CASE( "Load database with no spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{ "abc", "def" } );
auto db = completion::load_database( file.filename( ) );
auto expected =
std::vector< completion::channel >{ { "abc", "" }, { "def", "" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Load database with spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{
"abc", "def", "ghi jkl", "mno ", "pqr stu" } );
auto db = completion::load_database( file.filename( ) );
auto expected = std::vector< completion::channel >{ { "abc", "" },
{ "def", "" },
{ "ghi", "jkl" },
{ "mno", "" },
{ "pqr", "stu" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Basic completion behavior" )
{
completion::Database db;
db.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3",
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C",
{ "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
{ "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
};
{
......@@ -53,19 +224,35 @@ TEST_CASE( "Basic DB sorting" )
{
completion::Database input;
input.channels = {
"H0:VAC-SYSTEM_3", "H0:VAC-SYSTEM_1", "H1:SYS-SUB_C",
"H0:VAC-SYSTEM_2", "H1:SYS-SUB_A", "H1:SYS-SUB_B",
{ "H0:VAC-SYSTEM_3" }, { "H0:VAC-SYSTEM_1" }, { "H1:SYS-SUB_C" },
{ "H0:VAC-SYSTEM_2" }, { "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" },
};
completion::Database expected;
expected.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3",
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C",
{ "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
{ "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
};
completion::sort( input );
REQUIRE( input.channels.size( ) == expected.channels.size( ) );
for ( int i = 0; i < input.channels.size( ); ++i )
{
REQUIRE( input.channels[ i ] == expected.channels[ i ] );
REQUIRE( input.channels[ i ].name == expected.channels[ i ].name );
}
}
TEST_CASE( "Serialization of channel with an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc", "def" );
os << a;
REQUIRE( os.str( ) == "abc def" );
}
TEST_CASE( "Serialization of channel without an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc" );
os << a;
REQUIRE( os.str( ) == "abc" );
}
\ No newline at end of file
add_executable(ligo_channel_completion ligo_channel_completion.cpp)
target_link_libraries(ligo_channel_completion PUBLIC completion)
target_requires_cpp11(ligo_channel_completion PUBLIC)
install(TARGETS ligo_channel_completion DESTINATION lib)
\ No newline at end of file
install(TARGETS ligo_channel_completion DESTINATION lib)
add_executable(test_completion
tests/test_main.cc
tests/test_system_level.cc)
target_link_libraries(test_completion PUBLIC
pstream
catch2)
add_dependencies(test_completion ligo_channel_completion)
add_test(NAME completion_system_level
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_completion --program ${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion)
\ No newline at end of file
......@@ -7,8 +7,10 @@
#include "completion.hh"
#include <iostream>
#include <cstdlib>
#include <deque>
#include <iostream>
#include <stdexcept>
static const char CHAN_LIST_ENV[] = "CHAN_LIST";
......@@ -22,9 +24,12 @@ get_environment( const char* key )
struct option_t
{
option_t( ) : key( "" ), search( true ), resort( false ), abort( false )
option_t( )
: db_path( "" ), key( "" ), search( true ), resort( false ),
abort( false )
{
}
std::string db_path;
std::string key;
bool search;
bool resort;
......@@ -34,40 +39,71 @@ struct option_t
void
show_help( const char* prog )
{
std::cerr << "Usage " << prog << " option\n";
std::cerr << "Usage " << prog << " -d <path to db> options\n";
std::cerr << "Where option is one of:\n";
std::cerr << "\t-h - this help.\n";
std::cerr << "\t-d <path to db>- the path to the database to use. [A "
"required option]\n";
std::cerr << "\t-r - sort the database, dump results to stdout.\n";
std::cerr << "\t<search param> - search for the completion following the "
"given text.\n";
std::cerr << "\n\nNote: The environment variable CHAN_LIST must be set to "
"the path to text file\n";
std::cerr << "containing a sort list of channels. This is the channel "
"database used to do completions.\n";
}
option_t
parse_options( int argc, char* argv[] )
{
static const std::string opt_help( "-h" );
static const std::string opt_sort( "-r" );
static const std::string opt_db( "-d" );
option_t opts;
if ( argc > 1 )
{
if ( argc > 2 || std::strcmp( argv[ 1 ], "-h" ) == 0 )
std::deque< std::string > args;
auto next_opt = [&args]( ) -> std::string {
if ( args.empty( ) )
{
show_help( argv[ 0 ] );
opts.abort = true;
throw std::runtime_error( "Expecting another argument" );
}
else if ( std::strcmp( argv[ 1 ], "-r" ) == 0 )
{
opts.resort = true;
opts.search = false;
}
else
std::string results = args.front( );
args.pop_front( );
return results;
};
for ( int i = 1; i < argc; ++i )
{
args.emplace_back( argv[ i ] );
}
try
{
while ( !args.empty( ) )
{
opts.key = argv[ 1 ];
std::string cur = next_opt( );
if ( cur == opt_help )
{
show_help( argv[ 0 ] );
opts.abort = true;
break;
}
else if ( cur == opt_sort )
{
opts.resort = true;
opts.search = false;
}
else if ( cur == opt_db )
{
opts.db_path = next_opt( );
}
else
{
opts.key = cur;
}
}
}
catch ( std::runtime_error& err )