Commit c852524f authored by Jonathan Hanks's avatar Jonathan Hanks

Added systems level tests, split the search for EPICS/NDS.

This addes two new build/test requirements boost iterators and pstreams.

As there are several hundred thousand EPICS channels that are not recorded
in the frame, to do channel completion for EPICS needs more than the NDS
channel list.  This splits the system into supporting two databases.

Environment variables change from CHAN_LIST to EPICS_CHAN_LIST and
NDS_CHAN_LIST.

The database is allowed to have extra information following the channel
on each line, as long as a space is used to separate the extra data from
the channel.

Added tests against the ligo_channel_completion binary.
parent e0ad574e
A bash (and soon other) completion for LIGO channel names.
This provides a bash completion script along with a channel completion program.
A channel database is required. The database is a simple flat text file where
every line is a single channel name. The file must be sorted. The path to the
file is set via the 'CHAN_LIST' environment variable.
A set of channel databases are required. Each database is a simple flat text file where
every line is a single channel name. The file must be sorted.
If you have a list of channels you can have the application sort it for you
The path to the files is set via the 'EPICS_CHAN_LIST' and 'NDS_CHAN_LIST' environment variables.
As the names imply the database are for EPICS channels or Daq/NDS channels. This split is needed
as in the LIGO control room there are > 500k channels, with roughly half are available only as EPICS channels.
By specifically supporting EPICS channels then non-daq channels like string records can be completed against
for caget/caput/...
If you have a list of channels you can have the application sort it for you. Please note that
it is the bash scripts that use the EPICS_CHAN_LIST/NDS_CHAN_LIST variables, the completion engine
directly takes the database on the command line.
<pre>
# assuming that the channel database is
# located at a path specified by the
# CHAN_LIST environment variable
channel_completion -r > tmpfile
mv tmpfile "$CHAN_LIST"
# located at db.txt.
channel_completion -d db.txt -r > tmpfile
mv tmpfile db.txt
</pre>
Requirements:
* A C++11 compliant compiler.
* CMake 3+
* Boost iterator library
Debian requirements:
* build-essential
* bash-completion
* libboost-dev
* libpstreams-dev
Tested on:
* Debian 8 & 9
......@@ -43,9 +53,11 @@ make install
Components installed (Assuming a install prefix of /usr):
* a binary ligo_channel_completion to /usr/lib
* a shell script to the base completion libraries at /usr/share/bash-completion/completions
* a set shell script to the base completion libraries at /usr/share/bash-completion/completions
After the install is successful, start a new bash shell and use the channel completion.
After the install is successful, start a new bash shell and use the channel completion and make
sure that the EPICS_CHAN_LIST and NDS_CHAN_LIST environment variables are exported and refer
to proper databases.
<pre>
$ ndscope H&lt;tab&gt;
......
# Add commands that can use LIGO channel completion to this list
set (COMPLETE_THESE_COMMANDS
set (COMPLETE_THESE_NDS_COMMANDS
ndscope
)
set (COMPLETE_THESE_EPICS_COMMANDS
cdsutils
caget
caput
camonitor
)
set(COMPLETE_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_COMMANDS})
set(COMPLETE_COMMAND_REGISTRATION "${COMPLETE_COMMAND_REGISTRATION}
complete -F _channel_completion ${entry}")
set(COMPLETE_NDS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_NDS_COMMANDS})
set(COMPLETE_NDS_COMMAND_REGISTRATION "${COMPLETE_NDS_COMMAND_REGISTRATION}
complete -F _ligo_nds_channel_completion ${entry}")
endforeach()
set(COMPLETE_EPICS_COMMAND_REGISTRATION)
foreach(entry ${COMPLETE_THESE_EPICS_COMMANDS})
set(COMPLETE_EPICS_COMMAND_REGISTRATION "${COMPLETE_EPICS_COMMAND_REGISTRATION}
complete -F _ligo_epics_channel_completion ${entry}")
endforeach()
set(COMPLETE_SCRIPT_DIR "${CMAKE_INSTALL_PREFIX}/share/bash-completion/completions")
configure_file(ligo_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion @ONLY)
configure_file(ligo_nds_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion @ONLY)
configure_file(ligo_epics_channel_completion.in ${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion @ONLY)
configure_file(link_file.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion"
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_nds_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR})
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ligo_epics_channel_completion"
DESTINATION ${COMPLETE_SCRIPT_DIR})
install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/link_file.cmake)
# Copyright 2019 California Institute of Technology.
# You should have received a copy of the licensing terms for this
# software included in the file “LICENSE” located in the top-level
# directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt
_ligo_epics_channel_completion()
{
local cur
cur="${COMP_WORDS[COMP_CWORD]}"
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${EPICS_CHAN_LIST}") )
__ltrim_colon_completions "$cur"
return 0
}
@COMPLETE_EPICS_COMMAND_REGISTRATION@
\ No newline at end of file
......@@ -6,7 +6,7 @@
# directory of this package. If you did not, you can view a copy at
# http://dcc.ligo.org/M1500244/LICENSE.txt
_channel_completion()
_ligo_nds_channel_completion()
{
local cur
cur="${COMP_WORDS[COMP_CWORD]}"
......@@ -14,10 +14,10 @@ _channel_completion()
# see https://stackoverflow.com/questions/10528695/how-to-reset-comp-wordbreaks-without-affecting-other-completion-script/12495480#12495480
_get_comp_words_by_ref -n : cur
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}") )
COMPREPLY=( $(@CMAKE_INSTALL_PREFIX@/lib/ligo_channel_completion "${cur}" -d "${NDS_CHAN_LIST}") )
__ltrim_colon_completions "$cur"
return 0
}
@COMPLETE_COMMAND_REGISTRATION@
\ No newline at end of file
@COMPLETE_NDS_COMMAND_REGISTRATION@
\ No newline at end of file
......@@ -5,10 +5,19 @@
# variable (if present) which is found at run/install time, not at configuration time.
# ideas from https://stackoverflow.com/questions/35765106/symbolic-links-cmake
set (LINK_ENTRIES "@COMPLETE_THESE_COMMANDS@")
set (LINK_ENTRIES "@COMPLETE_THESE_NDS_COMMANDS@")
foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_channel_completion ${entry}
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_nds_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
)
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach()
set (LINK_ENTRIES "@COMPLETE_THESE_EPICS_COMMANDS@")
foreach (entry ${LINK_ENTRIES})
execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ligo_epics_channel_completion ${entry}
WORKING_DIRECTORY "$ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@"
)
message("-- Created symlink: for ${entry} in $ENV{DESTDIR}@COMPLETE_SCRIPT_DIR@")
endforeach()
\ No newline at end of file
find_package(Boost REQUIRED)
add_library(completion INTERFACE)
target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(completion INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} ${Boost_INCLUDEDIR})
target_requires_cpp11(completion INTERFACE)
add_executable(test_completion_lib
......
......@@ -11,32 +11,67 @@
#include <algorithm>
#include <iterator>
#include <fstream>
#include <ostream>
#include <string>
#include <vector>
#include <cstring>
#include <boost/iterator/transform_iterator.hpp>
namespace completion
{
typedef std::vector< std::string > string_list;
struct channel
{
std::string name;
std::string extra;
channel( ) = default;
channel( std::string name, std::string extra = "" )
: name( std::move( name ) ), extra( std::move( extra ) )
{
}
};
std::ostream&
operator<<( std::ostream& os, const channel& ch )
{
os << ch.name;
if ( !ch.extra.empty( ) )
{
os << " " << ch.extra;
}
return os;
}
struct Database
{
std::vector< std::string > channels;
std::vector< channel > channels;
};
namespace detail
{
const std::string&
project_name( const channel& entry )
{
return entry.name;
}
}
inline bool
simple_comparison( const std::string& a, const std::string& b )
simple_comparison( const channel& a, const channel& b )
{
return std::strcmp( a.c_str( ), b.c_str( ) ) < 0;
return std::strcmp( a.name.c_str( ), b.name.c_str( ) ) < 0;
}
inline bool
key_comparison( const std::string& key, const std::string& element )
key_comparison( const std::string& key, const channel& element )
{
int result =
std::strncmp( key.c_str( ), element.c_str( ), key.size( ) );
std::strncmp( key.c_str( ), element.name.c_str( ), key.size( ) );
return result < 0;
}
......@@ -48,17 +83,28 @@ namespace completion
std::string line;
while ( std::getline( input, line ) )
{
db.channels.emplace_back( line );
auto space = line.find( ' ' );
std::string extra;
if ( space < std::string::npos )
{
extra = line.substr( space + 1 );
line.resize( space );
}
db.channels.emplace_back( line, extra );
}
return db;
}
template < typename It >
string_list
search( It begin_it, It end_it, const std::string& key )
search( It begin_it_, It end_it_, const std::string& key )
{
string_list results;
auto begin_it =
boost::make_transform_iterator( begin_it_, detail::project_name );
auto end_it =
boost::make_transform_iterator( end_it_, detail::project_name );
auto lower = std::lower_bound( begin_it, end_it, key, key_comparison );
auto upper = std::upper_bound( lower, end_it, key, key_comparison );
auto key_size = key.size( );
......
......@@ -5,16 +5,186 @@
// directory of this package. If you did not, you can view a copy at
// http://dcc.ligo.org/M1500244/LICENSE.txt
#include <sstream>
#include "completion.hh"
#include "catch.hpp"
class temporary_fd
{
public:
explicit temporary_fd( std::string temp_dir ) : fd_( -1 ), name_( )
{
std::string path( std::move( temp_dir ) );
if ( !path.empty( ) )
{
if ( path.back( ) != '/' )
{
path.push_back( '/' );
}
}
path += "tmpfileXXXXXX";
std::vector< char > tmp;
tmp.reserve( path.size( ) + 1 );
std::copy( path.begin( ), path.end( ), std::back_inserter( tmp ) );
tmp.push_back( '\0' );
name_.clear( );
name_.reserve( tmp.size( ) );
fd_ = mkstemp( tmp.data( ) );
if ( fd_ >= 0 )
{
name_.append( tmp.data( ) );
}
}
temporary_fd( const temporary_fd& other ) = delete;
temporary_fd( temporary_fd&& other ) noexcept
: fd_( other.fd_ ),
name_( std::move( other.name_ ) )
{
other.fd_ = -1;
}
temporary_fd operator=( const temporary_fd& other ) = delete;
temporary_fd
operator=( temporary_fd&& other )
{
unlink( );
fd_ = other.fd_;
name_ = std::move( other.name_ );
other.fd_ = -1;
other.name_.clear( );
};
~temporary_fd( )
{
unlink( );
}
void
add_data( const std::string& data )
{
if ( *this )
{
size_t remaining = data.size( );
size_t copied = 0;
while ( remaining )
{
ssize_t count = write( fd_, data.data( ) + copied, remaining );
if ( count >= 0 )
{
remaining -= static_cast< size_t >( count );
copied += static_cast< size_t >( count );
}
else
{
auto err = errno;
if ( err == EAGAIN || err == EINTR )
{
continue;
}
throw std::runtime_error(
"Error writing data to temp file" );
}
}
}
}
void
add_data( const std::vector< std::string >& data )
{
for ( const auto& cur : data )
{
add_data( cur );
add_data( "\n" );
}
}
void
close( )
{
if ( fd_ >= 0 )
{
::close( fd_ );
}
fd_ = -1;
}
std::string
filename( ) const
{
return name_;
}
operator bool( ) const
{
return fd_ >= 0;
}
private:
void
unlink( )
{
close( );
if ( !name_.empty( ) )
{
::unlink( name_.c_str( ) );
}
name_.clear( );
}
int fd_;
std::string name_;
};
TEST_CASE( "Load empty database" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{} );
auto db = completion::load_database( file.filename( ) );
REQUIRE( db.channels.size( ) == 0 );
}
TEST_CASE( "Load database with no spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{ "abc", "def" } );
auto db = completion::load_database( file.filename( ) );
auto expected =
std::vector< completion::channel >{ { "abc", "" }, { "def", "" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Load database with spaces" )
{
temporary_fd file( "." );
file.add_data( std::vector< std::string >{
"abc", "def", "ghi jkl", "mno ", "pqr stu" } );
auto db = completion::load_database( file.filename( ) );
auto expected = std::vector< completion::channel >{ { "abc", "" },
{ "def", "" },
{ "ghi", "jkl" },
{ "mno", "" },
{ "pqr", "stu" } };
REQUIRE( db.channels.size( ) == expected.size( ) );
for ( int i = 0; i < db.channels.size( ); ++i )
{
REQUIRE( db.channels[ i ].name == expected[ i ].name );
REQUIRE( db.channels[ i ].extra == expected[ i ].extra );
}
}
TEST_CASE( "Basic completion behavior" )
{
completion::Database db;
db.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3",
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C",
{ "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
{ "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
};
{
......@@ -53,19 +223,35 @@ TEST_CASE( "Basic DB sorting" )
{
completion::Database input;
input.channels = {
"H0:VAC-SYSTEM_3", "H0:VAC-SYSTEM_1", "H1:SYS-SUB_C",
"H0:VAC-SYSTEM_2", "H1:SYS-SUB_A", "H1:SYS-SUB_B",
{ "H0:VAC-SYSTEM_3" }, { "H0:VAC-SYSTEM_1" }, { "H1:SYS-SUB_C" },
{ "H0:VAC-SYSTEM_2" }, { "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" },
};
completion::Database expected;
expected.channels = {
"H0:VAC-SYSTEM_1", "H0:VAC-SYSTEM_2", "H0:VAC-SYSTEM_3",
"H1:SYS-SUB_A", "H1:SYS-SUB_B", "H1:SYS-SUB_C",
{ "H0:VAC-SYSTEM_1" }, { "H0:VAC-SYSTEM_2" }, { "H0:VAC-SYSTEM_3" },
{ "H1:SYS-SUB_A" }, { "H1:SYS-SUB_B" }, { "H1:SYS-SUB_C" },
};
completion::sort( input );
REQUIRE( input.channels.size( ) == expected.channels.size( ) );
for ( int i = 0; i < input.channels.size( ); ++i )
{
REQUIRE( input.channels[ i ] == expected.channels[ i ] );
REQUIRE( input.channels[ i ].name == expected.channels[ i ].name );
}
}
TEST_CASE( "Serialization of channel with an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc", "def" );
os << a;
REQUIRE( os.str( ) == "abc def" );
}
TEST_CASE( "Serialization of channel without an extra portion" )
{
std::ostringstream os;
completion::channel a( "abc" );
os << a;
REQUIRE( os.str( ) == "abc" );
}
\ No newline at end of file
add_executable(ligo_channel_completion ligo_channel_completion.cpp)
target_link_libraries(ligo_channel_completion PUBLIC completion)
target_requires_cpp11(ligo_channel_completion PUBLIC)
install(TARGETS ligo_channel_completion DESTINATION lib)
\ No newline at end of file
install(TARGETS ligo_channel_completion DESTINATION lib)
add_executable(test_completion
tests/test_main.cc
tests/test_system_level.cc)
target_link_libraries(test_completion PUBLIC
pstream
catch2)
add_dependencies(test_completion ligo_channel_completion)
add_test(NAME completion_system_level
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_completion --program ${CMAKE_CURRENT_BINARY_DIR}/ligo_channel_completion)
\ No newline at end of file
......@@ -7,8 +7,10 @@
#include "completion.hh"
#include <iostream>
#include <cstdlib>
#include <deque>
#include <iostream>
#include <stdexcept>
static const char CHAN_LIST_ENV[] = "CHAN_LIST";
......@@ -22,9 +24,12 @@ get_environment( const char* key )
struct option_t
{
option_t( ) : key( "" ), search( true ), resort( false ), abort( false )
option_t( )
: db_path( "" ), key( "" ), search( true ), resort( false ),
abort( false )
{
}
std::string db_path;
std::string key;
bool search;
bool resort;
......@@ -34,40 +39,71 @@ struct option_t
void
show_help( const char* prog )
{
std::cerr << "Usage " << prog << " option\n";
std::cerr << "Usage " << prog << " -d <path to db> options\n";
std::cerr << "Where option is one of:\n";
std::cerr << "\t-h - this help.\n";
std::cerr << "\t-d <path to db>- the path to the database to use. [A "
"required option]\n";
std::cerr << "\t-r - sort the database, dump results to stdout.\n";
std::cerr << "\t<search param> - search for the completion following the "
"given text.\n";
std::cerr << "\n\nNote: The environment variable CHAN_LIST must be set to "
"the path to text file\n";
std::cerr << "containing a sort list of channels. This is the channel "
"database used to do completions.\n";
}
option_t
parse_options( int argc, char* argv[] )
{
static const std::string opt_help( "-h" );
static const std::string opt_sort( "-r" );
static const std::string opt_db( "-d" );
option_t opts;
if ( argc > 1 )
{
if ( argc > 2 || std::strcmp( argv[ 1 ], "-h" ) == 0 )
std::deque< std::string > args;
auto next_opt = [&args]( ) -> std::string {
if ( args.empty( ) )
{
show_help( argv[ 0 ] );
opts.abort = true;
throw std::runtime_error( "Expecting another argument" );
}
else if ( std::strcmp( argv[ 1 ], "-r" ) == 0 )
{
opts.resort = true;
opts.search = false;
}
else
std::string results = args.front( );
args.pop_front( );
return results;
};
for ( int i = 1; i < argc; ++i )
{
args.emplace_back( argv[ i ] );
}
try
{
while ( !args.empty( ) )
{
opts.key = argv[ 1 ];
std::string cur = next_opt( );
if ( cur == opt_help )
{
show_help( argv[ 0 ] );
opts.abort = true;
break;
}
else if ( cur == opt_sort )
{
opts.resort = true;
opts.search = false;
}
else if ( cur == opt_db )
{
opts.db_path = next_opt( );
}
else
{
opts.key = cur;
}
}
}
catch ( std::runtime_error& err )
{
opts.abort = true;
}
return opts;
}
......@@ -79,12 +115,7 @@ main( int argc, char* argv[] )
{
exit( 1 );
}
std::string channel_db_path = get_environment( CHAN_LIST_ENV );
if ( channel_db_path.empty( ) )
{
exit( 1 );
}
completion::Database db = completion::load_database( channel_db_path );
completion::Database db = completion::load_database( opts.db_path );
if ( opts.search )
{
completion::string_list choices;
......@@ -107,9 +138,10 @@ main( int argc, char* argv[] )
else if ( opts.resort )
{
completion::sort( db );
std::copy( db.channels.begin( ),
db.channels.end( ),
std::ostream_iterator< std::string >( std::cout, "\n" ) );
std::copy(
db.channels.begin( ),
db.channels.end( ),
std::ostream_iterator< completion::channel >( std::cout, "\n" ) );
}
return 0;
}
\ No newline at end of file
// this main function is almost entirely taken from
// https://github.com/catchorg/Catch2/blob/master/docs/own-main.md#top
// The additional changes are
// Copyright 2019 California Institute of Technology.
// You should have received a copy of the licensing terms for this
// software included in the file “LICENSE” located in the top-level
// directory of this package. If you did not, you can view a copy at
// http://dcc.ligo.org/M1500244/LICENSE.txt
#include <string>
#define CATCH_CONFIG_RUNNER
#include "catch.hpp"
std::string test_program_name;
int
main( int argc, char* argv[] )
{
Catch::Session session; // There must be exactly one instance
// Build a new parser on top of Catch's
using namespace Catch::clara;
auto cli = session.cli( ) // Get Catch's composite command line parser
| Opt( test_program_name,
"The program program to test" ) // bind variable to a new option,
// with a hint string
[ "--program" ] // the option
// names it
// will
// respond to
( "The test program" ); // description string for the help output
// Now pass the new composite back to Catch so it uses that
session.cli( cli );
// Let Catch (using Clara) parse the command line
int returnCode = session.applyCommandLine( argc, argv );