Skip to content
Snippets Groups Projects
Commit c31ef474 authored by Jonathan Hanks's avatar Jonathan Hanks
Browse files

Merge branch 'rawtrend-cli' into 'master' 

Add CLI utilities for working with raw trend files

See merge request cds/advligorts!203
parents 576e835d f3fd5c3c
No related branches found
No related tags found
2 merge requests!439RCG 5.0 release fro deb 10,!203Add CLI utilities for working with raw trend files
Hide whitespace changes
Inline Side-by-side
src/nds/CMakeLists.txt
+ 52
− 1
View file @ c31ef474
......@@ -40,6 +40,57 @@ configure_file(config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../gds)
add_executable(extracttrend extracttrend.cc)
target_compile_definitions(extracttrend PRIVATE -D_REENTRANT -DDAQD_CPP11)
target_compile_definitions(extracttrend PRIVATE ${NDS_COMMON_DEFINES})
target_compile_definitions(extracttrend PRIVATE -DDAQD_CPP11)
target_include_directories(extracttrend PRIVATE
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/client
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src/zlib
${CMAKE_CURRENT_SOURCE_DIR}/../include
${Boost_INCLUDE_DIRS}
)
target_link_libraries(extracttrend PRIVATE
args
ldastools::framecpp
${Boost_LIBRARIES})
add_executable(mergetrend mergetrend.cc)
target_compile_definitions(mergetrend PRIVATE -D_REENTRANT -DDAQD_CPP11)
target_compile_definitions(mergetrend PRIVATE ${NDS_COMMON_DEFINES})
target_compile_definitions(mergetrend PRIVATE -DDAQD_CPP11)
target_include_directories(mergetrend PRIVATE
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/client
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src/zlib
${CMAKE_CURRENT_SOURCE_DIR}/../include
${Boost_INCLUDE_DIRS}
)
target_link_libraries(mergetrend PRIVATE
args
ldastools::framecpp
${Boost_LIBRARIES})
add_executable(validaterawtrend validaterawtrend.cc)
target_compile_definitions(validaterawtrend PRIVATE -D_REENTRANT -DDAQD_CPP11)
target_compile_definitions(validaterawtrend PRIVATE ${NDS_COMMON_DEFINES})
target_compile_definitions(validaterawtrend PRIVATE -DDAQD_CPP11)
target_include_directories(validaterawtrend PRIVATE
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/client
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src
${CMAKE_CURRENT_SOURCE_DIR}/framelib/src/zlib
${CMAKE_CURRENT_SOURCE_DIR}/../include
${Boost_INCLUDE_DIRS}
)
target_link_libraries(validaterawtrend PRIVATE
args
ldastools::framecpp
${Boost_LIBRARIES})
add_executable(nds ${NDS_SRC})
target_compile_definitions(nds PRIVATE -D_REENTRANT -DDAQD_CPP11)
target_compile_definitions(nds PRIVATE ${NDS_COMMON_DEFINES})
......@@ -60,4 +111,4 @@ target_link_libraries(nds PRIVATE
install(TARGETS nds DESTINATION bin)
# -L/opt/ldas-tools-al-2.5.5/lib -L/opt/ldas-tools-framecpp-2.5.2/lib -lframecpp -lframecpp8 -lframecpp7 -lframecpp6 -lframecpp4 -lframecpp3 -lframecppcmn -lldastoolsal -lz -lbz2 -ldl -lnsl
\ No newline at end of file
# -L/opt/ldas-tools-al-2.5.5/lib -L/opt/ldas-tools-framecpp-2.5.2/lib -lframecpp -lframecpp8 -lframecpp7 -lframecpp6 -lframecpp4 -lframecpp3 -lframecppcmn -lldastoolsal -lz -lbz2 -ldl -lnsl
src/nds/extracttrend.cc 0 → 100644
+ 237
− 0
View file @ c31ef474
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <sys/wait.h>
#include <iostream>
#include <fstream>
#include <list>
#include <set>
#include <map>
#include <string>
#include <memory>
#include "nds.hh"
#include "io.h"
#include "daqd_net.hh"
#include "../daqd/crc8.cc"
using namespace CDS_NDS;
using namespace std;
int nds_log_level=4; // Controls volume of log messages
int _debug = 4;
typedef struct trend_block_on_disk_t {
/// These two ints represent a double
unsigned int min;
unsigned int min2;
/// These two ints represent a double
unsigned int max;
unsigned int max2;
unsigned int n;
/// These two ints represent a double
unsigned int rms;
unsigned int rms2;
/// These two ints represent a double
unsigned int mean;
unsigned int mean2;
/// Assign with the in-memory trend structure
void operator=(const trend_block_t& t) {
memcpy(&min, &t.min, 2*sizeof(unsigned int));
memcpy(&max, &t.max, 2*sizeof(unsigned int));
n = t.n;
memcpy(&rms, &t.rms, 2*sizeof(unsigned int));
memcpy(&mean, &t.mean, 2*sizeof(unsigned int));
}
} trend_block_on_disk_t;
typedef struct raw_trend_disk_record_struct {
unsigned int gps;
trend_block_on_disk_t tb;
} raw_trend_disk_record_struct;
// Fixed record size to be independent of 32/64 bit difference
// struct raw_trend_record_struct does not represent the layout of data on disk
// only in memory.
#define RAW_TREND_REC_SIZE 40
static unsigned long
find_offs (unsigned int gps, int fd, unsigned long start, unsigned long structs)
{
lseek (fd, RAW_TREND_REC_SIZE * (start + structs/2), SEEK_SET);
raw_trend_record_struct ss;
int nread = read (fd, &ss, RAW_TREND_REC_SIZE);
//if (sizeof(raw_trend_record_struct) == 48) {
// we must be on the 64-bit computer
//}
// Do not need to shuffle the data since ss.gps is the first 4 bytes always
if (1 == structs) {
if (gps < ss.gps + 30)
return start;
else
return start + 1;
}
int begin;
if (gps > ss.gps)
begin = start + structs/2;
else
begin = start;
DEBUG1(cerr << "recursion: " << gps << " " << ss.gps << " " << begin << " " << (structs+1)/2 << endl);
return find_offs (gps, fd, begin, (structs+1)/2);
}
inline void
byteswap(char *image_ptr, int chb)
{
char a[8];
switch(chb) {
case 2:
a[0] = *image_ptr;
*image_ptr = image_ptr[1];
image_ptr[1] = a[0];
break;
case 4:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
image_ptr[0] = a[3];
image_ptr[1] = a[2];
image_ptr[2] = a[1];
image_ptr[3] = a[0];
break;
case 8:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
a[4] = image_ptr[4];
a[5] = image_ptr[5];
a[6] = image_ptr[6];
a[7] = image_ptr[7];
image_ptr[0] = a[7];
image_ptr[1] = a[6];
image_ptr[2] = a[5];
image_ptr[3] = a[4];
image_ptr[4] = a[3];
image_ptr[5] = a[2];
image_ptr[6] = a[1];
image_ptr[7] = a[0];
break;
}
}
int printUntil(int fd, int gpsEnd, int lastReadGPS, raw_trend_disk_record_struct *sd) {
int nread = 0;
do {
if (sd->gps <= lastReadGPS) {
fprintf(stderr, "Skipping record: time goes backwards: %d -> %d\n", sd->gps, lastReadGPS);
} else if ((lastReadGPS - sd->gps) %60 != 0) {
fprintf(stderr, "Skipping record: time increased by an interval != 60seconds: %d\n", lastReadGPS - sd->gps);
} else {
//fprintf(stderr, "Wrote data at %d\n", sd->gps);
fwrite(sd, RAW_TREND_REC_SIZE, 1, stdout);
lastReadGPS = sd->gps;
}
nread = read (fd, sd, RAW_TREND_REC_SIZE);
} while (sd->gps < gpsEnd && nread != 0);
return nread;
}
int main (int argc, char **argv) {
multimap<string, int> fname_channel_mmap; // file_name -> index of the signal in the user request
typedef multimap<string, int>::const_iterator FCMI;
set<string> file_set; // to keep file names
map<string, list<data_span> > data_span_map; // file_name -> list of data spans
typedef map<string, list<data_span> >::const_iterator DSMI;
typedef map<string, list<data_span> >::iterator NCDSMI;
typedef list<data_span>::const_iterator DSI;
typedef list<data_span>::iterator NCDSI;
typedef list<mapping_data_span>::const_iterator MDSI;
typedef list<mapping_data_span>::iterator NCMDSI;
typedef set<string>::const_iterator SSI;
if (argc != 5) {
fprintf(stderr, "Usage: %s <infile> <gpsStart> <gpsEnd> <outfile>\n", argv[0]);
exit(1);
}
string infile = argv[1];
int gpsstart = atoi(argv[2]);
int gpsend = atoi(argv[3]);
string outfile = argv[4];
// For every file construct a list of data spans (index, gps, length), `data_span_map'
// Each file contains data with gaps. This we end up with the list of data spans.
int fd = open (infile.c_str(), O_RDONLY);
int fdout = open (outfile.c_str(), O_WRONLY|O_TRUNC|O_CREAT);
if (fd < 0) {
system_log(1, "Couldn't open raw minute trend file `%s' for reading; errno %d", infile.c_str(), errno);
} else if (fdout < 0) {
system_log(1, "Couldn't open raw minute trend file `%s' for writing; errno %d", outfile.c_str(), errno);
} else {
struct stat st;
struct stat st_out;
int res = fstat (fd, &st);
int resout = fstat (fdout, &st_out);
if (res) {
system_log(1, "Couldn't stat raw minute trend file `%s'; errno %d", infile.c_str(), errno);
} else if (resout) {
system_log(1, "Couldn't stat raw minute trend file `%s'; errno %d", outfile.c_str(), errno);
} else {
int structs = st.st_size / RAW_TREND_REC_SIZE;
int structsout = st_out.st_size / RAW_TREND_REC_SIZE;
raw_trend_disk_record_struct sd;
DEBUG1(cerr << infile << ": length " << st.st_size << "; structs " << structs << endl);
if (st.st_size%RAW_TREND_REC_SIZE) {
system_log (1, "WARNING: filesize of %s isn't multiple of %d (record size)\n", infile.c_str(), RAW_TREND_REC_SIZE);
exit(1);
} else {
int nread = 0;
int nwrite = 0;
int endFound = 0;
int lastReadGPS = 0;
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
while (sd.gps <= gpsend && !endFound) {
if (nread == 0) {
fprintf(stderr, "EOF on input file at %d.\n", sd.gps);
endFound = 1;
} else if (sd.gps > gpsend) {
fprintf(stderr, "GPS End reached on src file at %d.\n", sd.gps);
endFound = 1;
} else if (sd.gps+60 >= gpsstart && sd.gps <= gpsend ) {
nwrite = write (fdout, &sd, RAW_TREND_REC_SIZE);
fprintf(stdout, "Wrote GPS record %ld\n", sd.gps);
} else {
fprintf(stdout, "Skipping GPS record %ld\n", sd.gps);
}
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
}
}
}
close (fd);
close (fdout);
}
exit(0);
}
src/nds/mergetrend.cc 0 → 100644
+ 301
− 0
View file @ c31ef474
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <sys/wait.h>
#include <iostream>
#include <fstream>
#include <list>
#include <set>
#include <map>
#include <string>
#include <memory>
#include "nds.hh"
#include "io.h"
#include "daqd_net.hh"
#include "../daqd/crc8.cc"
using namespace CDS_NDS;
using namespace std;
int nds_log_level=4; // Controls volume of log messages
int _debug = 4;
typedef struct trend_block_on_disk_t {
/// These two ints represent a double
unsigned int min;
unsigned int min2;
/// These two ints represent a double
unsigned int max;
unsigned int max2;
unsigned int n;
/// These two ints represent a double
unsigned int rms;
unsigned int rms2;
/// These two ints represent a double
unsigned int mean;
unsigned int mean2;
/// Assign with the in-memory trend structure
void operator=(const trend_block_t& t) {
memcpy(&min, &t.min, 2*sizeof(unsigned int));
memcpy(&max, &t.max, 2*sizeof(unsigned int));
n = t.n;
memcpy(&rms, &t.rms, 2*sizeof(unsigned int));
memcpy(&mean, &t.mean, 2*sizeof(unsigned int));
}
} trend_block_on_disk_t;
typedef struct raw_trend_disk_record_struct {
unsigned int gps;
trend_block_on_disk_t tb;
} raw_trend_disk_record_struct;
// Fixed record size to be independent of 32/64 bit difference
// struct raw_trend_record_struct does not represent the layout of data on disk
// only in memory.
#define RAW_TREND_REC_SIZE 40
static unsigned long
find_offs (unsigned int gps, int fd, unsigned long start, unsigned long structs)
{
lseek (fd, RAW_TREND_REC_SIZE * (start + structs/2), SEEK_SET);
raw_trend_record_struct ss;
int nread = read (fd, &ss, RAW_TREND_REC_SIZE);
//if (sizeof(raw_trend_record_struct) == 48) {
// we must be on the 64-bit computer
//}
// Do not need to shuffle the data since ss.gps is the first 4 bytes always
if (1 == structs) {
if (gps < ss.gps + 30)
return start;
else
return start + 1;
}
int begin;
if (gps > ss.gps)
begin = start + structs/2;
else
begin = start;
DEBUG1(cerr << "recursion: " << gps << " " << ss.gps << " " << begin << " " << (structs+1)/2 << endl);
return find_offs (gps, fd, begin, (structs+1)/2);
}
inline void
byteswap(char *image_ptr, int chb)
{
char a[8];
switch(chb) {
case 2:
a[0] = *image_ptr;
*image_ptr = image_ptr[1];
image_ptr[1] = a[0];
break;
case 4:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
image_ptr[0] = a[3];
image_ptr[1] = a[2];
image_ptr[2] = a[1];
image_ptr[3] = a[0];
break;
case 8:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
a[4] = image_ptr[4];
a[5] = image_ptr[5];
a[6] = image_ptr[6];
a[7] = image_ptr[7];
image_ptr[0] = a[7];
image_ptr[1] = a[6];
image_ptr[2] = a[5];
image_ptr[3] = a[4];
image_ptr[4] = a[3];
image_ptr[5] = a[2];
image_ptr[6] = a[1];
image_ptr[7] = a[0];
break;
}
}
int printUntil(int fd, int gpsEnd, int lastReadGPS, raw_trend_disk_record_struct *sd) {
int nread = 0;
do {
if (sd->gps <= lastReadGPS) {
fprintf(stderr, "Skipping record: time goes backwards: %d -> %d\n", sd->gps, lastReadGPS);
} else if ((lastReadGPS - sd->gps) %60 != 0) {
fprintf(stderr, "Skipping record: time increased by an interval != 60seconds: %d\n", lastReadGPS - sd->gps);
} else {
//fprintf(stderr, "Wrote data at %d\n", sd->gps);
fwrite(sd, RAW_TREND_REC_SIZE, 1, stdout);
lastReadGPS = sd->gps;
}
nread = read (fd, sd, RAW_TREND_REC_SIZE);
} while (sd->gps < gpsEnd && nread != 0);
return nread;
}
int main (int argc, char **argv) {
multimap<string, int> fname_channel_mmap; // file_name -> index of the signal in the user request
typedef multimap<string, int>::const_iterator FCMI;
set<string> file_set; // to keep file names
map<string, list<data_span> > data_span_map; // file_name -> list of data spans
typedef map<string, list<data_span> >::const_iterator DSMI;
typedef map<string, list<data_span> >::iterator NCDSMI;
typedef list<data_span>::const_iterator DSI;
typedef list<data_span>::iterator NCDSI;
typedef list<mapping_data_span>::const_iterator MDSI;
typedef list<mapping_data_span>::iterator NCMDSI;
typedef set<string>::const_iterator SSI;
if (argc != 6) {
fprintf(stderr, "Usage: %s <channel> <path1> <path2> <gpsStart> <gpsEnd>\n", argv[0]);
exit(1);
}
string channel = argv[1];
string path = argv[2];
string pathfiller = argv[3];
int gpsstart = atoi(argv[4]);
int gpsend = atoi(argv[5]);
file_set.insert(channel);
// For every file construct a list of data spans (index, gps, length), `data_span_map'
// Each file contains data with gaps. This we end up with the list of data spans.
for (SSI p = file_set.begin (); p != file_set.end (); p++) {
//string fname_str = path + "/" + crc8_str(p->c_str()) + "/" +*p;
string fname_str = path + "/" + *p;
const char *fname = fname_str.c_str ();
int fd = open (fname, O_RDONLY);
string fillername_str = pathfiller + "/" + crc8_str(p->c_str()) + "/" +*p;
const char *fillername = fillername_str.c_str ();
int fdfiller = open (fillername, O_RDONLY);
if (fd < 0) {
system_log(1, "Couldn't open raw minute trend file `%s' for reading; errno %d", fname, errno);
} else if (fdfiller < 0) {
system_log(1, "Couldn't open raw minute trend file `%s' for reading; errno %d", fillername, errno);
} else {
struct stat st;
struct stat stfiller;
int res = fstat (fd, &st);
int resfiller = fstat (fdfiller, &stfiller);
if (res) {
system_log(1, "Couldn't stat raw minute trend file `%s'; errno %d", fname, errno);
} else if (resfiller) {
system_log(1, "Couldn't stat raw minute trend file `%s'; errno %d", fillername, errno);
} else {
int structs = st.st_size / RAW_TREND_REC_SIZE;
int structsfiller = stfiller.st_size / RAW_TREND_REC_SIZE;
raw_trend_disk_record_struct sd;
raw_trend_disk_record_struct sdfiller;
DEBUG1(cerr << fname << ": length " << st.st_size << "; structs " << structs << endl);
if (st.st_size%RAW_TREND_REC_SIZE) {
system_log (1, "WARNING: filesize of %s isn't multiple of %d (record size)\n", fname, RAW_TREND_REC_SIZE);
exit(1);
} else if (stfiller.st_size%RAW_TREND_REC_SIZE) {
system_log (1, "WARNING: filesize of %s isn't multiple of %d (record size)\n", fillername, RAW_TREND_REC_SIZE);
exit(1);
} else {
int nread = 0;
int nreadfiller = 0;
int endFound = 0;
int lastReadGPS = 0;
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
nreadfiller = read (fdfiller, &sdfiller, RAW_TREND_REC_SIZE);
fprintf(stderr, "Start of src: %d\n", sd.gps);
fprintf(stderr, "Start of filler: %d\n", sdfiller.gps);
while (!endFound) {
if (nread == 0) {
/*
* EOF on source file. Continue filling in data from
* the filler file.
*/
fprintf(stderr, "EOF on input file at %d. Completing with filler file.\n", sd.gps);
nreadfiller = printUntil(fdfiller, gpsend, lastReadGPS, &sdfiller);
lastReadGPS = sdfiller.gps;
} else if (nreadfiller == 0) {
/*
* EOF on filler file. Continue filling in data from
* the source file.
*/
fprintf(stderr, "EOF on filler file at %d. Completing with input file.\n", sdfiller.gps);
nread = printUntil(fd, gpsend, lastReadGPS, &sd);
lastReadGPS = sd.gps;
} else if (sd.gps == sdfiller.gps) {
if (sd.gps - lastReadGPS > 60) {
fprintf(stderr, "Unfillable gap from %d to %d (%d s)\n", lastReadGPS, sd.gps, sd.gps - lastReadGPS);
}
if (sd.tb.n > sdfiller.tb.n) {
fprintf(stderr, "Filled partial trend from src (%d vs. %d) at %d\n", sd.tb.n, sdfiller.tb.n, sdfiller.gps);
fwrite(&sd, RAW_TREND_REC_SIZE, 1, stdout);
lastReadGPS = sd.gps;
} else if (sd.tb.n < sdfiller.tb.n) {
fprintf(stderr, "Filled partial trend from filler (%d vs. %d) at %d\n", sd.tb.n, sdfiller.tb.n, sdfiller.gps);
fwrite(&sdfiller, RAW_TREND_REC_SIZE, 1, stdout);
lastReadGPS = sdfiller.gps;
} else {
//fprintf(stderr, "Wrote data at %d\n", sd.gps);
fwrite(&sd, RAW_TREND_REC_SIZE, 1, stdout);
lastReadGPS = sd.gps;
}
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
nreadfiller = read (fdfiller, &sdfiller, RAW_TREND_REC_SIZE);
} else if (sd.gps < sdfiller.gps) {
fprintf(stderr, "Filling reverse gap from %d to %d (%d s)\n", sd.gps, sdfiller.gps, sdfiller.gps - sd.gps);
nread = printUntil(fd, sdfiller.gps, lastReadGPS, &sd);
lastReadGPS = sd.gps;
} else {
fprintf(stderr, "Filling gap from %d to %d (%d s)\n", sdfiller.gps, sd.gps, sd.gps - sdfiller.gps);
nreadfiller = printUntil(fdfiller, sd.gps, lastReadGPS, &sdfiller);
lastReadGPS = sdfiller.gps;
}
if ((nread == 0 || sd.gps > gpsend) && (nreadfiller == 0 || sdfiller.gps > gpsend)) {
if (nread == 0) {
fprintf(stderr, "EOF on input file at %d.\n", sd.gps);
}
if (nreadfiller == 0) {
fprintf(stderr, "EOF on filler file at %d.\n", sdfiller.gps);
}
if (sd.gps > gpsend) {
fprintf(stderr, "GPS End reached on src file at %d.\n", sd.gps);
}
if (sdfiller.gps > gpsend) {
fprintf(stderr, "GPS End reached on filler file at %d.\n", sdfiller.gps);
}
endFound = 1;
}
}
}
}
close (fd);
close (fdfiller);
}
}
exit(0);
}
src/nds/validaterawtrend.cc 0 → 100644
+ 337
− 0
View file @ c31ef474
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <sys/wait.h>
#include <unistd.h>
#include <iostream>
#include <fstream>
#include <list>
#include <set>
#include <map>
#include <string>
#include <memory>
#include "nds.hh"
#include "io.h"
#include "daqd_net.hh"
#include "../daqd/crc8.cc"
using namespace CDS_NDS;
using namespace std;
int nds_log_level=4; // Controls volume of log messages
int _debug = 4;
typedef struct trend_block_on_disk_t {
/// These two ints represent a double
unsigned int min;
unsigned int min2;
/// These two ints represent a double
unsigned int max;
unsigned int max2;
unsigned int n;
/// These two ints represent a double
unsigned int rms;
unsigned int rms2;
/// These two ints represent a double
unsigned int mean;
unsigned int mean2;
/// Assign with the in-memory trend structure
void operator=(const trend_block_t& t) {
memcpy(&min, &t.min, 2*sizeof(unsigned int));
memcpy(&max, &t.max, 2*sizeof(unsigned int));
n = t.n;
memcpy(&rms, &t.rms, 2*sizeof(unsigned int));
memcpy(&mean, &t.mean, 2*sizeof(unsigned int));
}
} trend_block_on_disk_t;
typedef struct raw_trend_disk_record_struct {
unsigned int gps;
trend_block_on_disk_t tb;
} raw_trend_disk_record_struct;
// Fixed record size to be independent of 32/64 bit difference
// struct raw_trend_record_struct does not represent the layout of data on disk
// only in memory.
#define RAW_TREND_REC_SIZE 40
static unsigned long
find_offs (unsigned int gps, int fd, unsigned long start, unsigned long structs)
{
lseek (fd, RAW_TREND_REC_SIZE * (start + structs/2), SEEK_SET);
raw_trend_record_struct ss;
int nread = read (fd, &ss, RAW_TREND_REC_SIZE);
//if (sizeof(raw_trend_record_struct) == 48) {
// we must be on the 64-bit computer
//}
// Do not need to shuffle the data since ss.gps is the first 4 bytes always
if (1 == structs) {
if (gps < ss.gps + 30)
return start;
else
return start + 1;
}
int begin;
if (gps > ss.gps)
begin = start + structs/2;
else
begin = start;
DEBUG1(cerr << "recursion: " << gps << " " << ss.gps << " " << begin << " " << (structs+1)/2 << endl);
return find_offs (gps, fd, begin, (structs+1)/2);
}
inline void
byteswap(char *image_ptr, int chb)
{
char a[8];
switch(chb) {
case 2:
a[0] = *image_ptr;
*image_ptr = image_ptr[1];
image_ptr[1] = a[0];
break;
case 4:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
image_ptr[0] = a[3];
image_ptr[1] = a[2];
image_ptr[2] = a[1];
image_ptr[3] = a[0];
break;
case 8:
a[0] = image_ptr[0];
a[1] = image_ptr[1];
a[2] = image_ptr[2];
a[3] = image_ptr[3];
a[4] = image_ptr[4];
a[5] = image_ptr[5];
a[6] = image_ptr[6];
a[7] = image_ptr[7];
image_ptr[0] = a[7];
image_ptr[1] = a[6];
image_ptr[2] = a[5];
image_ptr[3] = a[4];
image_ptr[4] = a[3];
image_ptr[5] = a[2];
image_ptr[6] = a[1];
image_ptr[7] = a[0];
break;
}
}
void print_usage(const char *progname) {
fprintf(stderr, "Usage: %s [-g] [-v] [-e] [-s] [-f] [-G maxgap] <filename>\n", progname);
fprintf(stderr, " -g Print each gap\n");
fprintf(stderr, " -v Print each data record\n");
fprintf(stderr, " -e Exit the first time an error is found\n");
fprintf(stderr, " -s Do not print the summary of gaps and time spans\n");
fprintf(stderr, " -f Purge invalid data and write the result to stdout\n");
fprintf(stderr, " -G maxgap Maximum allowed size of a gap.\n");
fprintf(stderr, " -m minGPS Minimum allowed GPS timestamp on data records.\n");
fprintf(stderr, " Defaults to timestamp on first data record\n");
fprintf(stderr, " -M maxGPS Maximum allowed GPS timestamp on data records.\n");
fprintf(stderr, " Defaults to timestamp on final data record\n");
}
int main (int argc, char **argv) {
multimap<string, int> fname_channel_mmap; // file_name -> index of the signal in the user request
typedef multimap<string, int>::const_iterator FCMI;
set<string> file_set; // to keep file names
map<string, list<data_span> > data_span_map; // file_name -> list of data spans
typedef map<string, list<data_span> >::const_iterator DSMI;
typedef map<string, list<data_span> >::iterator NCDSMI;
typedef list<data_span>::const_iterator DSI;
typedef list<data_span>::iterator NCDSI;
typedef list<mapping_data_span>::const_iterator MDSI;
typedef list<mapping_data_span>::iterator NCMDSI;
typedef set<string>::const_iterator SSI;
int opt;
int verbose = 0;
int exit_on_error = 0;
int print_gaps = 0;
int print_summary = 1;
int fix_file = 0;
int max_gap = 0;
int lastRecordGPS = 0;
int startGPS = 0;
while ((opt = getopt(argc, argv, "sgvefG:m:M:")) != -1) {
switch (opt) {
case 'v':
verbose++;
break;
case 'g':
print_gaps++;
break;
case 's':
print_summary = 0;
break;
case 'e':
exit_on_error++;
break;
case 'f':
fix_file++;
break;
case 'G':
max_gap=atoi(optarg);
break;
case 'm':
startGPS=atoi(optarg);
break;
case 'M':
lastRecordGPS=atoi(optarg);
break;
default:
print_usage(argv[0]);
exit(1);
}
}
if (optind >= argc) {
print_usage(argv[0]);
exit(1);
}
int errorFound = 0;
string channel = argv[optind];
file_set.insert(channel);
// For every file construct a list of data spans (index, gps, length), `data_span_map'
// Each file contains data with gaps. This we end up with the list of data spans.
for (SSI p = file_set.begin (); p != file_set.end (); p++) {
string fname_str = *p;
const char *fname = fname_str.c_str ();
int fd = open (fname, O_RDONLY);
if (fd < 0) {
system_log(1, "Couldn't open raw minute trend file `%s' for reading; errno %d", fname, errno);
} else {
struct stat st;
int res = fstat (fd, &st);
if (res) {
system_log(1, "Couldn't stat raw minute trend file `%s'; errno %d", fname, errno);
} else {
int structs = st.st_size / RAW_TREND_REC_SIZE;
raw_trend_disk_record_struct sd;
DEBUG1(cerr << fname << ": length " << st.st_size << "; structs " << structs << endl);
if (st.st_size%RAW_TREND_REC_SIZE) {
system_log (1, "WARNING: filesize of %s isn't multiple of %d (record size)\n", fname, RAW_TREND_REC_SIZE);
}
int nread = 0;
if (lastRecordGPS == 0) {
lseek (fd, -RAW_TREND_REC_SIZE, SEEK_END);
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
if (nread != 0) {
lastRecordGPS = sd.gps;
}
lseek (fd, 0, SEEK_SET);
}
int numRecords = 0;
int numGaps = 0;
int gapSpan = 0;
int prevGPS = 0;
int longestGap = 0;
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
while (nread != 0) {
numRecords++;
if (startGPS == 0) {
startGPS = sd.gps;
if (startGPS > lastRecordGPS) {
fprintf(stderr, "Timestamp of final record is less than timestamp of first record: %d vs. %d\n", startGPS, lastRecordGPS);
if (exit_on_error) {
exit (1);
}
}
}
if (sd.gps % 60 != 0) {
fprintf(stderr, "Record %d does not start at a 60-second boundary: %d (off by %d)\n", numRecords, sd.gps, sd.gps%60);
errorFound++;
if (exit_on_error) {
exit (1);
}
} else if (sd.gps < startGPS || sd.gps > lastRecordGPS) {
fprintf(stderr, "Record %d has a timestamp outside the bounds of the file: %d not within %d -> %d\n", numRecords, sd.gps, startGPS, lastRecordGPS);
errorFound++;
if (exit_on_error) {
exit (1);
}
} else if (prevGPS > 0 && sd.gps < prevGPS) {
fprintf(stderr, "Time goes backwards at record %d: %d -> %d\n", numRecords, prevGPS, sd.gps);
errorFound++;
if (exit_on_error) {
exit (1);
}
} else if (prevGPS > 0 && max_gap > 0 && sd.gps - prevGPS > max_gap) {
fprintf(stderr, "Gap found between %d and %d lasting %d s, exceeded max allowed gap size of %d.\n", prevGPS, sd.gps, sd.gps - prevGPS, max_gap);
if (sd.gps - prevGPS > longestGap) {
longestGap = sd.gps - prevGPS;
}
errorFound++;
if (exit_on_error) {
exit (1);
}
} else if (prevGPS > 0 && sd.gps - prevGPS > 60) {
if (print_gaps >= 1) {
fprintf(stderr, "Gap found between %d and %d lasting %d s\n", prevGPS, sd.gps, sd.gps - prevGPS);
}
if (sd.gps - prevGPS > longestGap) {
longestGap = sd.gps - prevGPS;
}
numGaps++;
gapSpan += sd.gps - prevGPS;
prevGPS = sd.gps;
if (fix_file && sd.gps - prevGPS <= max_gap) {
fwrite(&sd, RAW_TREND_REC_SIZE, 1, stdout);
}
} else if (prevGPS > 0 && sd.gps - prevGPS != 60) {
fprintf(stderr, "Bad gap size found at record %d between %d and %d lasting %d s\n", numRecords, prevGPS, sd.gps, sd.gps - prevGPS);
errorFound++;
if (exit_on_error) {
exit (1);
}
prevGPS = sd.gps;
} else {
if (fix_file) {
fwrite(&sd, RAW_TREND_REC_SIZE, 1, stdout);
}
prevGPS = sd.gps;
}
if (verbose) {
fprintf(stderr, "%d: %d min:%d+%d max:%d+%d rms:%d+%d mean:%d+%d n:%d\n", numRecords, sd.gps, sd.tb.min, sd.tb.min2, sd.tb.max, sd.tb.max2, sd.tb.rms, sd.tb.rms2, sd.tb.mean, sd.tb.mean2, sd.tb.n);
}
nread = read (fd, &sd, RAW_TREND_REC_SIZE);
}
if (print_summary) {
fprintf(stderr, "%d records with %d gaps spanning %d seconds found\n", numRecords, numGaps, gapSpan);
fprintf(stderr, "Max gap length is %d\n", longestGap);
fprintf(stderr, "GPS time range: %d to %d\n", startGPS, prevGPS);
}
}
close (fd);
}
}
if (errorFound) {
exit(1);
}
exit(0);
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment