From 613308e6909def2cb8bb15a34c46a938f45cc730 Mon Sep 17 00:00:00 2001
From: Jonathan Hanks <jonathan.hanks@ligo.org>
Date: Thu, 22 Aug 2019 14:16:41 -0700
Subject: [PATCH] Updates to answer review notes.
Fix usage message in local_dc.
Remove commented out code in the makefile and associated .c code
Looking for variables with ix_ prefix and removed an usused bit.
---
src/ix_stream/Makefile | 38 -
src/ix_stream/dix_recv.c | 6 +-
src/ix_stream/dix_xmit.c | 13 +-
src/ix_stream/ix_dc_xmit.c | 197 ----
src/ix_stream/ix_multi_stream.c | 539 -----------
src/ix_stream/ix_rcvr.c | 348 -------
src/ix_stream/ix_rcvr_threads.c | 420 ---------
src/ix_stream/mx2ix.c | 829 -----------------
src/ix_stream/mx2rcv.c | 823 -----------------
src/ix_stream/mx_rcv.cc | 901 ------------------
src/ix_stream/mx_rcv_simple.cc | 631 -------------
src/ix_stream/zmq_rcv_ix_xmit.c | 1335 ---------------------------
src/ix_stream/zmq_rcv_ix_xmit.cc | 1471 ------------------------------
src/local_dc/local_dc.c | 4 +-
14 files changed, 8 insertions(+), 7547 deletions(-)
delete mode 100644 src/ix_stream/ix_dc_xmit.c
delete mode 100644 src/ix_stream/ix_multi_stream.c
delete mode 100644 src/ix_stream/ix_rcvr.c
delete mode 100644 src/ix_stream/ix_rcvr_threads.c
delete mode 100644 src/ix_stream/mx2ix.c
delete mode 100644 src/ix_stream/mx2rcv.c
delete mode 100644 src/ix_stream/mx_rcv.cc
delete mode 100644 src/ix_stream/mx_rcv_simple.cc
delete mode 100644 src/ix_stream/zmq_rcv_ix_xmit.c
delete mode 100644 src/ix_stream/zmq_rcv_ix_xmit.cc
diff --git a/src/ix_stream/Makefile b/src/ix_stream/Makefile
index 6605ddbf8..1e8c0101b 100644
--- a/src/ix_stream/Makefile
+++ b/src/ix_stream/Makefile
@@ -43,31 +43,6 @@ rfm.o: ../drv/rfm.c
crc.o: ../drv/crc.c
$(CC) $(CFLAGS) -c $< -o $@
-# ix_rcvr: ix_rcvr.c
-# $(CC) $(CFLAGS) ix_rcvr.c -c -o ix_rcvr.o
-# $(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
-# $(CC) $(FLAGS) -o ix_rcvr ix_rcvr.o rfm.o -L $(API_LIB_PATH) -lsisci
-# sync
-#
-# ix_rcvr_threads: ix_rcvr_threads.c
-# $(CC) $(CFLAGS) ix_rcvr_threads.c -c -o ix_rcvr_threads.o
-# $(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
-# $(CC) $(FLAGS) -o ix_rcvr_threads ix_rcvr_threads.o rfm.o -L $(API_LIB_PATH) -lsisci
-# sync
-#
-# ix_dc_xmit: ix_dc_xmit.c
-# $(CC) $(CFLAGS) ix_dc_xmit.c -c -o ix_dc_xmit.o
-# $(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
-# $(CC) $(FLAGS) -o ix_dc_xmit ix_dc_xmit.o rfm.o -L $(API_LIB_PATH) -lsisci
-# sync
-
-# mx2ix: mx2ix.c
-# $(CC) $(CFLAGS) -I../zmq_stream -I/opt/open-mx/include mx2ix.c -c -o mx2ix.o
-# $(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
-# $(CC) $(CFLAGS) -I../zmq_stream ../zmq_stream/simple_pv.c -c -o simple_pv.o
-# $(CC) $(FLAGS) -o mx2ix mx2ix.o rfm.o simple_pv.o -L $(API_LIB_PATH) -L/opt/open-mx/lib -lsisci -lmyriexpress -lpthread
-# sync
-
dix_xmit: dix_xmit.c
$(CC) $(CFLAGS) -I../zmq_stream -I/opt/open-mx/include dix_xmit.c -c -o dix_xmit.o
$(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
@@ -75,25 +50,12 @@ dix_xmit: dix_xmit.c
$(CC) $(FLAGS) -o dix_xmit dix_xmit.o rfm.o simple_pv.o -L $(API_LIB_PATH) -L/opt/open-mx/lib -lmyriexpress -lpthread
sync
-# zmq_rcv_ix_xmit: zmq_rcv_ix_xmit.c
-# $(CC) $(CFLAGS) -I../zmq_stream zmq_rcv_ix_xmit.c -c -o zmq_rcv_ix_xmit.o
-# $(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
-# $(CC) $(CFLAGS) -I../zmq_stream ../zmq_stream/simple_pv.c -c -o simple_pv.o
-# $(CC) $(CFLAGS) ../zmq_stream/zmq_transport.c -c -o zmq_transport.o
-# $(CC) $(CFLAGS) ../zmq_stream/dc_utils.c -c -o dc_utils.o
-# $(CC) $(FLAGS) -o zmq_rcv_ix_xmit zmq_rcv_ix_xmit.o rfm.o zmq_transport.o dc_utils.o simple_pv.o -L $(API_LIB_PATH) -lsisci -lzmq -lpthread
-# sync
-
dix_recv: dix_recv.c
$(CC) $(CFLAGS) dix_recv.c -c -o dix_recv.o
$(CC) $(CFLAGS) ../drv/rfm.c -c -o rfm.o
$(CC) $(FLAGS) -o dix_recv dix_recv.o rfm.o -L $(API_LIB_PATH) -lsisci
sync
-# datarate: datarate.c
-# $(CC) $(CFLAGS) datarate.c -c -o datarate.o
-# $(CC) $(FLAGS) -o datarate datarate.o rfm.o param.o crc.o
-
CPPFLAGS = -I../common -I$(top_srcdir)/common -I$(top_srcdir)/common/bsd -DNO_RTL -D_XOPEN_SOURCE -D_BSD_SOURCE
LDFLAGS = -L/opt/open-mx/lib -lmyriexpress -lpthread -g
diff --git a/src/ix_stream/dix_recv.c b/src/ix_stream/dix_recv.c
index 0fc5c41f6..35e20a61d 100644
--- a/src/ix_stream/dix_recv.c
+++ b/src/ix_stream/dix_recv.c
@@ -36,14 +36,14 @@ Usage( )
{
printf( "Usage of dix_recv:\n" );
printf( "dix_recv -g <group>\n" );
- printf( "-b <name> : Destination buffer name (default local_dc)\n" );
- printf( "-m <value> : Size in MB of the destination buffer [20-100] "
+ printf( " -b <name> : Destination buffer name (default local_dc)\n" );
+ printf( " -m <value> : Size in MB of the destination buffer [20-100] "
"(default=100)\n" );
printf( " -a <value> : Local adapter number (default %d)\n",
localAdapterNo );
printf( " -g <value> : Reflective group identifier (0..5))\n" );
printf( " -v <value> : Diagnostics level (0..1) \n" );
- printf( " -h : This helpscreen\n" );
+ printf( " -h : This help screen\n" );
printf( "\n" );
}
diff --git a/src/ix_stream/dix_xmit.c b/src/ix_stream/dix_xmit.c
index d5a547154..09688f7a3 100644
--- a/src/ix_stream/dix_xmit.c
+++ b/src/ix_stream/dix_xmit.c
@@ -50,7 +50,7 @@ void
usage( )
{
fprintf( stderr,
- "Usage: dix_xmit [args] -m shared memory size -g IX "
+ "Usage: dix_ix_xmit [args] -m shared memory size -g IX "
"channel \n" );
fprintf( stderr, "-l filename - log file name\n" );
fprintf( stderr, "-b buffer name - Input buffer [local_dc]\n" );
@@ -357,7 +357,7 @@ main( int argc, char** argv )
int64_t cur_ref_time = 0;
int festatus = 0;
int pv_festatus = 0;
- int ix_xmit_stop = 0;
+
SimplePV pvs[] = {
{
"RECV_MIN_MS",
@@ -569,7 +569,7 @@ main( int argc, char** argv )
missed_flag <<= 1;
}
- if ( xmitData && !ix_xmit_stop )
+ if ( xmitData )
{
if ( sendLength > IX_BLOCK_SIZE )
{
@@ -621,13 +621,6 @@ main( int argc, char** argv )
// network
SCIFlush( sequence, SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY );
}
-
- if ( ix_xmit_stop )
- {
- ix_xmit_stop--;
- if ( ix_xmit_stop == 0 )
- fprintf( stderr, "Restarting Dolphin Xmit\n" );
- }
}
sprintf( dcstatus, "%ld ", ets );
for ( ii = 0; ii < mytotaldcu; ii++ )
diff --git a/src/ix_stream/ix_dc_xmit.c b/src/ix_stream/ix_dc_xmit.c
deleted file mode 100644
index 163584f6c..000000000
--- a/src/ix_stream/ix_dc_xmit.c
+++ /dev/null
@@ -1,197 +0,0 @@
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include "testlib.h"
-#include <malloc.h>
-#include <signal.h>
-#include <unistd.h>
-
-#include "../drv/crc.c"
-#include "../include/daq_core.h"
-
-
-#define __CDECL
-
-#define DO_HANDSHAKE 0
-
-// #define MY_DCU_OFFSET 0x1a00000
-#define MY_DCU_OFFSET 0x00000
-#define MY_IPC_OFFSET (MY_DCU_OFFSET + 0x8000)
-#define MY_GDS_OFFSET (MY_DCU_OFFSET + 0x9000)
-#define MY_DAT_OFFSET (MY_DCU_OFFSET + 0xa000)
-
-#include "./dolphin_common.c"
-extern void *findSharedMemorySize(char *,int);
-static volatile int keepRunning = 1;
-
-
-/*********************************************************************************/
-/* U S A G E */
-/* */
-/*********************************************************************************/
-
-void Usage()
-{
- printf("Usage of ix_dc_xmit:\n");
- printf("ix_multi_stream -n <nodes> -g <group> -m <models> \n");
- printf(" -a <value> : Local adapter number (default %d)\n", localAdapterNo);
- printf(" -g <value> : Reflective group identifier (0..3))\n");
- printf(" -v <value> : Diagnostics level (0..1))\n");
- printf(" -h : This helpscreen\n");
- printf("\n");
-}
-
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-
-/*********************************************************************************/
-/* M A I N */
-/* */
-/*********************************************************************************/
-
-int __CDECL
-main(int argc,char *argv[])
-{
- int counter;
- int ii;
- char *mywriteaddr;
- int myCrc;
- static const int xmitDataOffset = MY_DAT_OFFSET + sizeof(struct daq_multi_cycle_header_t);
- static const int header_size = sizeof(daq_multi_dcu_header_t);
- char *sysname;
- daq_multi_dcu_data_t *ixDataBlock;
- int do_verbose = 0;
- int sendLength = 0;
- daq_multi_cycle_header_t *xmitHeader;
- // daq_multi_dcu_data_t *xmitData;
- int lastCycle = 0;
- int new_cycle;
- char *nextData;
-
- FILE *mydiags;
-
- printf("\n %s compiled %s : %s\n\n",argv[0],__DATE__,__TIME__);
-
- if (argc<2) {
- printf("Exiting here \n");
- Usage();
- return(-1);
- }
-
- /* Get the parameters */
- while ((counter = getopt(argc, argv, "g:a:v:")) != EOF)
- switch(counter) {
-
- case 'g':
- segmentId = atoi(optarg);
- break;
-
- case 'a':
- localAdapterNo = atoi(optarg);
- continue;
-
- case 'v':
- do_verbose = atoi(optarg);
- continue;
-
- case 'h':
- Usage();
- return(0);
- }
-
- // Attach to local shared memory
- char *ifo = (char *)findSharedMemorySize("ifo",100);
- // Set pointer to ifo data header
- daq_multi_cycle_header_t *ifo_header = (daq_multi_cycle_header_t *)ifo;
- // Set pointer to ifo data block
- char *ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
-
- // Connect to Dolphin
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- // Set pointer to xmit header in Dolphin xmit data area.
- mywriteaddr = (char *)writeAddr;
- mywriteaddr += MY_DAT_OFFSET;
- xmitHeader = (daq_multi_cycle_header_t *)mywriteaddr;
-
- // Trap control C exit
- signal(SIGINT,intHandler);
-
- // Sync to cycle zero in ifo shared memory (FE receive data area)
- for (;ifo_header->curCycle;) usleep(1000);
- printf("Found cycle zero = %d\n",ifo_header->curCycle);
- printf("Found cycle size = %d\n",ifo_header->cycleDataSize);
- int cyclesize = ifo_header->cycleDataSize;
-
-
- do{
- // Wait for cycle count update in ifo shared memory
- // Check every 2 milliseconds
- do{
- usleep(2000);
- new_cycle = ifo_header->curCycle;
- } while (new_cycle == lastCycle && keepRunning);
- // Save cycle number of last received
- lastCycle = new_cycle;
- // Move pointer to proper data cycle in ifo memory
- nextData = (char *)ifo_data;
- nextData += cyclesize * new_cycle;
- ixDataBlock = (daq_multi_dcu_data_t *)nextData;
- // Set the xmit data length to be header size plus data size
- sendLength = header_size + ixDataBlock->header.dataBlockSize;
-
- // Print some diagnostics to file
- if(new_cycle == 0 && do_verbose > 0)
- {
- mydiags = fopen("./ix_diags.txt","w");
- if(mydiags) {
- fprintf(mydiags,"%d %d %d\n", ixDataBlock->header.dcuTotalModels,
- (sendLength * 16),
- ixDataBlock->header.dcuheader[0].timeSec);
- }
- fclose(mydiags);
- }
-
- // Print some diagnostics to terminal
- if(new_cycle == 0 && do_verbose == 2)
- {
- printf("\t\tNum DCU = %d\n", ixDataBlock->header.dcuTotalModels);
- printf("\t\tNew Size = %d\n", ixDataBlock->header.dataBlockSize);
- printf("\t\tTime = %d\n", ixDataBlock->header.dcuheader[0].timeSec);
- printf("\t\tSend Size = %d\n", sendLength);
- printf("\t\tCycle Size = %d\n", DAQ_TRANSIT_DC_DATA_BLOCK_SIZE);
- }
-
- // WRITEDATA to Dolphin Network
- SCIMemCpy(sequence,ixDataBlock, remoteMap,xmitDataOffset,sendLength,memcpyFlag,&error);
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- return error;
- }
- // Calculate data CRC checksum
- myCrc = crc_ptr((char *)ixDataBlock, sendLength, 0);
- myCrc = crc_len(sendLength, myCrc);
-
- // Set data header information
- xmitHeader->maxCycle = ifo_header->maxCycle;
- xmitHeader->cycleDataSize = sendLength;
- xmitHeader->msgcrc = myCrc;
- // Send cycle last as indication of data ready for receivers
- xmitHeader->curCycle = ifo_header->curCycle;
- // Have to flush the buffers to make data go onto Dolphin network
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- } while(keepRunning);
-
- // Cleanup the Dolphin connections
- error = dolphin_closeout();
- // Exit
- return SCI_ERR_OK;
-}
diff --git a/src/ix_stream/ix_multi_stream.c b/src/ix_stream/ix_multi_stream.c
deleted file mode 100644
index a043f3980..000000000
--- a/src/ix_stream/ix_multi_stream.c
+++ /dev/null
@@ -1,539 +0,0 @@
-
-/*********************************************************************************
- * *
- * Copyright (C) 1993 - 2015 *
- * Dolphin Interconnect Solutions AS *
- * *
- * This program is free software; you can redistribute it and/or modify *
- * it under the terms of the GNU General Public License as published by *
- * the Free Software Foundation; either version 2 of the License, *
- * or (at your option) any later version. *
- * *
- * This program is distributed in the hope that it will be useful, *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
- * GNU General Public License for more details. *
- * *
- * You should have received a copy of the GNU General Public License *
- * along with this program; if not, write to the Free Software *
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
- * *
- * *
- *********************************************************************************/
-
-
-/*********************************************************************************/
-/* */
-/* This program demonstrates the use of the SISCI Reflective Memory */
-/* functionality available with the Dolphin Express DX technology. */
-/* */
-/* This functionality is not available for the SCI technology. */
-/* */
-/*********************************************************************************/
-
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include "testlib.h"
-#include <malloc.h>
-#include <signal.h>
-#include <unistd.h>
-
-#include "../drv/crc.c"
-#include "../include/daqmap.h"
-#include "../include/drv/fb.h"
-#include "../include/daq_core.h"
-
-
-#define __CDECL
-
-#define DAQ_RDY_MAX_WAIT 80
-
-
-#define DO_HANDSHAKE 0
-
-// #define MY_DCU_OFFSET 0x1a00000
-#define MY_DCU_OFFSET 0x00000
-#define MY_IPC_OFFSET (MY_DCU_OFFSET + 0x1000)
-#define MY_GDS_OFFSET (MY_DCU_OFFSET + 0x9000)
-#define MY_DAT_OFFSET (MY_DCU_OFFSET + 0xa000)
-
-#include "./dolphin_common.c"
-extern void *findSharedMemory(char *);
-
-static struct rmIpcStr *shmIpcPtr[128];
-static char *shmDataPtr[128];
-static struct cdsDaqNetGdsTpNum *shmTpTable[128];
-static const int header_size = sizeof(struct daq_fe_header_t);
-int *drIntData;
-static const int buf_size = DAQ_DCU_BLOCK_SIZE;
-char modelnames[DAQ_TRANSIT_MAX_DCU][64];
-char *sysname;
-int modelrates[DAQ_TRANSIT_MAX_DCU];
-daq_fe_data_t ixDataBlock;
-char *daqbuffer = (char *) &ixDataBlock;
-char buffer[1024000];
-char *zbuffer;
-int do_verbose = 1;
-int sendLength = 0;
-static volatile int keepRunning = 1;
-
-unsigned int loops = 170;
-
-/*********************************************************************************/
-/* U S A G E */
-/* */
-/*********************************************************************************/
-
-void Usage()
-{
- printf("Usage of ix_multi_stream:\n");
- printf("ix_multi_stream -n <nodes> -g <group> -m <models> \n");
- printf(" -a <value> : Local adapter number (default %d)\n", localAdapterNo);
- printf(" -s <value> : Segment size (default %d)\n", segmentSize);
- printf(" -g <value> : Reflective group identifier (0..5))\n");
- printf(" -n <value> : Number of receivers\n");
- printf(" -l <value> : Loops to execute (default %d)\n", loops);
- printf(" -h : This helpscreen\n");
- printf("\n");
-}
-
-int sync2zero(struct rmIpcStr *ipcPtr) {
- int lastCycle = 0;
-
- // Find cycle zero
- for (;ipcPtr->cycle;) usleep(1000);
- return(lastCycle);
-}
-
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-int getmodelrate( char *modelname, char *gds_tp_dir) {
- int rate = 0;
- char gdsfile[128];
- int ii = 0;
- FILE *f = 0;
- char *token = 0;
- char *search = "=";
- char line[80];
- char *s = 0;
- char *s1 = 0;
-
- if (gds_tp_dir) {
- sprintf(gdsfile, "%s/tpchn_%s.par", gds_tp_dir, modelname);
- } else {
- /// Need to get IFO and SITE info from environment variables.
- s = getenv("IFO");
- for (ii = 0; s[ii] != '\0'; ii++) {
- if (isupper(s[ii])) s[ii] = (char) tolower(s[ii]);
- }
- s1 = getenv("SITE");
- for (ii = 0; s1[ii] != '\0'; ii++) {
- if (isupper(s1[ii])) s1[ii] = (char) tolower(s1[ii]);
- }
- sprintf(gdsfile, "/opt/rtcds/%s/%s/target/gds/param/tpchn_%s.par", s1, s, modelname);
- }
- f = fopen(gdsfile, "rt");
- if (!f) return 0;
- while(fgets(line,80,f) != NULL) {
- line[strcspn(line, "\n")] = 0;
- if (strstr(line, "datarate") != NULL) {
- token = strtok(line, search);
- token = strtok(NULL, search);
- if (!token) continue;
- while (*token && *token == ' ') {
- ++token;
- }
- rate = atoi(token);
- break;
- }
- }
- fclose(f);
-
- return rate;
-}
-
-int waitServers(int nodes,sci_sequence_t sequence,volatile unsigned int *readAddr,volatile unsigned int *writeAddr)
-{
-int node_offset;
-int value;
-
- /* Lets wait for the servers to write CMD_READY */
- printf("Wait for %d receivers ...\n", nodes);
-
- for (node_offset=1; node_offset <= nodes;node_offset++){
- int wait_loops = 0;
-
- do {
- value = (*(readAddr+IX_SYNC_OFFSET+node_offset));
- wait_loops++;
- } while (value != CMD_READY && keepRunning);
- }
- printf("Client received CMD_READY from all nodes \n\n",value);
-
- /* Lets write CMD_READY to offset 0 to signal all servers to go on. */
- *(writeAddr+IX_SYNC_OFFSET) = CMD_READY;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-
- if(!keepRunning) return 1;
- else return 0;
-}
-
-sci_error_t send_via_reflective_memory(int nsys)
-{
- unsigned int value;
- unsigned int written_value = 0;
- int verbose = 1;
- int node_offset;
- volatile unsigned int *myreadAddr;
- volatile unsigned int *mywriteAddr;
-
-
- int timeout;
- int myCrc;
- int do_wait = 1;
- int daqStatBit[2];
- daqStatBit[0] = 1;
- daqStatBit[1] = 2;
- int reftimeerror = 0;
- int refcycle;
- int reftimeSec;
- int reftimeNSec;
- int dataTPLength;
- int status;
-
- myreadAddr = (unsigned int *)readAddr;
- mywriteAddr = (unsigned int *)writeAddr;
-
-
-
- status = waitServers(nodes,sequence,readAddr,writeAddr);
- if (status) return SCI_ERR_OK;
-
- mywriteAddr += 256;
- myreadAddr += 256;
-
- written_value=1;
- int ii;
- int new_cycle = 0;
- int lastCycle = 0;
-unsigned char *dataBuff;
-
- int sync2iop = 1;
-
-
-// for (;shmIpcPtr[0]->cycle;) usleep(1000);
-// printf("Found cycle zero in client\n");
- do {
-
- if(sync2iop) {
- printf("Syncing to IOP\n");
- lastCycle = sync2zero(shmIpcPtr[0]);
- sync2iop = 0;
- printf("Found cycle zero\n");
- }
- timeout = 0;
- // Wait for a new 1/16Hz DAQ data cycle
- do{
- usleep(1000);
- new_cycle = shmIpcPtr[0]->cycle;
- timeout += 1;
- }while (new_cycle == lastCycle && timeout < DAQ_RDY_MAX_WAIT);
- if(timeout >= DAQ_RDY_MAX_WAIT) {
- sync2iop = 1;
- lastCycle = sync2zero(shmIpcPtr[0]);
- printf("Iop model not running\n");
- }
- if(sync2iop) continue;
- // IOP will be first model ready
- // Need to wait for 2K models to reach end of their cycled
- usleep((do_wait * 1000));
-
- // Print diags in verbose mode
- if(new_cycle == 0 && !do_verbose) {
- printf("\nTime = %d-%d size = %d\n",shmIpcPtr[0]->bp[lastCycle].timeSec,shmIpcPtr[0]->bp[lastCycle].timeNSec,sendLength);
- printf("\tCycle = ");
- for(ii=0;ii<nsys;ii++) printf("\t\t%d",ixDataBlock.header.dcuheader[ii].cycle);
- printf("\n\tTimeSec = ");
- for(ii=0;ii<nsys;ii++) printf("\t%d",ixDataBlock.header.dcuheader[ii].timeSec);
- printf("\n\tTimeNSec = ");
- for(ii=0;ii<nsys;ii++) printf("\t\t%d",ixDataBlock.header.dcuheader[ii].timeNSec);
- printf("\n\tDataSize = ");
- for(ii=0;ii<nsys;ii++) printf("\t\t%d",ixDataBlock.header.dcuheader[ii].dataBlockSize);
- printf("\n\tTPCount = ");
- for(ii=0;ii<nsys;ii++) printf("\t\t%d",ixDataBlock.header.dcuheader[ii].tpCount);
- printf("\n\tTPSize = ");
- for(ii=0;ii<nsys;ii++) printf("\t\t%d",ixDataBlock.header.dcuheader[ii].tpBlockSize);
- printf("\n\n ");
- }
-
- // Increment the local DAQ cycle counter
- lastCycle ++;
- lastCycle %= 16;
-
-
- // Set pointer to 0MQ message data block
- zbuffer = (char *)&ixDataBlock.dataBlock[0];
- // Initialize data send length to size of message header
- sendLength = header_size;
- // Set number of FE models that have data in this message
- ixDataBlock.header.dcuTotalModels = nsys;
- ixDataBlock.header.dataBlockSize = 0;
- // Loop thru all FE models
- for (ii=0;ii<nsys;ii++) {
- reftimeerror = 0;
- // Set heartbeat monitor for return to DAQ software
- if (lastCycle == 0) shmIpcPtr[ii]->reqAck ^= daqStatBit[0];
- // Set DCU ID in header
- ixDataBlock.header.dcuheader[ii].dcuId = shmIpcPtr[ii]->dcuId;
- // Set DAQ .ini file CRC checksum
- ixDataBlock.header.dcuheader[ii].fileCrc = shmIpcPtr[ii]->crc;
- // Set 1/16Hz cycle number
- ixDataBlock.header.dcuheader[ii].cycle = shmIpcPtr[ii]->cycle;
- if(ii == 0) refcycle = shmIpcPtr[ii]->cycle;
- // Set GPS seconds
- ixDataBlock.header.dcuheader[ii].timeSec = shmIpcPtr[ii]->bp[lastCycle].timeSec;
- if (ii == 0) reftimeSec = shmIpcPtr[ii]->bp[lastCycle].timeSec;
- // Set GPS nanoseconds
- ixDataBlock.header.dcuheader[ii].timeNSec = shmIpcPtr[ii]->bp[lastCycle].timeNSec;
- if (ii == 0) reftimeNSec = shmIpcPtr[ii]->bp[lastCycle].timeNSec;
- if (ii != 0 && reftimeSec != shmIpcPtr[ii]->bp[lastCycle].timeSec)
- reftimeerror = 1;;
- if (ii != 0 && reftimeNSec != shmIpcPtr[ii]->bp[lastCycle].timeNSec)
- reftimeerror |= 2;;
- if(reftimeerror) {
- ixDataBlock.header.dcuheader[ii].cycle = refcycle;
- // printf("Timing error model %d\n",ii);
- // Set Status -- Need to update for models not running
- ixDataBlock.header.dcuheader[ii].status = 0xbad;
- // Indicate size of data block
- ixDataBlock.header.dcuheader[ii].dataBlockSize = 0;
- ixDataBlock.header.dcuheader[ii].tpBlockSize = 0;
- ixDataBlock.header.dcuheader[ii].tpCount = 0;
- } else {
- // Set Status -- Need to update for models not running
- ixDataBlock.header.dcuheader[ii].status = 2;
- // Indicate size of data block
- ixDataBlock.header.dcuheader[ii].dataBlockSize = shmIpcPtr[ii]->dataBlockSize;
- // Prevent going beyond MAX allowed data size
- if (ixDataBlock.header.dcuheader[ii].dataBlockSize > DAQ_DCU_BLOCK_SIZE)
- ixDataBlock.header.dcuheader[ii].dataBlockSize = DAQ_DCU_BLOCK_SIZE;
-
- ixDataBlock.header.dcuheader[ii].tpCount = (unsigned int)shmTpTable[ii]->count;
- ixDataBlock.header.dcuheader[ii].tpBlockSize = sizeof(float) * modelrates[ii] * ixDataBlock.header.dcuheader[ii].tpCount;
- // Prevent going beyond MAX allowed data size
- if (ixDataBlock.header.dcuheader[ii].tpBlockSize > DAQ_DCU_BLOCK_SIZE)
- ixDataBlock.header.dcuheader[ii].tpBlockSize = DAQ_DCU_BLOCK_SIZE;
-
- memcpy(&(ixDataBlock.header.dcuheader[ii].tpNum[0]),
- &(shmTpTable[ii]->tpNum[0]),
- sizeof(int)*ixDataBlock.header.dcuheader[ii].tpCount);
-
- // Set pointer to dcu data in shared memory
- dataBuff = (char *)(shmDataPtr[ii] + lastCycle * buf_size);
- // Copy data from shared memory into local buffer
- dataTPLength = ixDataBlock.header.dcuheader[ii].dataBlockSize + ixDataBlock.header.dcuheader[ii].tpBlockSize;
- memcpy((void *)zbuffer, dataBuff, dataTPLength);
-
-
- // Calculate CRC on the data and add to header info
- myCrc = crc_ptr((char *)zbuffer, ixDataBlock.header.dcuheader[ii].dataBlockSize, 0); // .crc is crcLength
- myCrc = crc_len(ixDataBlock.header.dcuheader[ii].dataBlockSize, myCrc);
- ixDataBlock.header.dcuheader[ii].dataCrc = myCrc;
-
- // Increment the 0mq data buffer pointer for next FE
- zbuffer += dataTPLength;
- // Increment the 0mq message size with size of FE data block
- sendLength += dataTPLength;
- // Increment the data block size for the message
- ixDataBlock.header.dataBlockSize += (unsigned int)dataTPLength;
-
- // Update heartbeat monitor to DAQ code
- if (lastCycle == 0) shmIpcPtr[ii]->reqAck ^= daqStatBit[1];
- }
- }
-
- // Copy data to combined message buffer
- memcpy(buffer,daqbuffer,sendLength);
- // Send Data
-
- if (!verbose) {
- printf("Send broadcast message (value %d) to %d available nodes ...\n",written_value,nodes);
- }
-
- /* Lets write the value to offset 0 */
- // WRITEDATA
- SCIMemCpy(sequence,buffer, remoteMap,MY_DAT_OFFSET,sendLength,memcpyFlag,&error);
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- return error;
- }
-
- *mywriteAddr = written_value;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-
- //printf("Writing cycle %d with size %d \n",lastCycle,sendLength);
- /* Lets wait for the servers to write the written value +1 */
-
- for (node_offset=1; node_offset <= nodes;node_offset++){
- int wait_loops = 0;
-
- do {
- value = (*(myreadAddr+node_offset));
- wait_loops++;
- if ((wait_loops % 10000000)==0){
- printf("Value = %d (expected = %d) delayed from rank %d - after %u reads\n", value, written_value+1, node_offset, wait_loops);
- return SCI_ERR_OK;
- }
- } while (value != written_value+1);
- }
-
- if (!verbose) {
- printf("Received broadcast ack %d from all nodes \n\n",value);
- }
- written_value++;
-
- } while (keepRunning); /* do this number of loops */
-
- printf("\n***********************************************************\n\n");
-
-
- return SCI_ERR_OK;
-}
-
-
-/*********************************************************************************/
-/* M A I N */
-/* */
-/*********************************************************************************/
-
-int __CDECL
-main(int argc,char *argv[])
-{
- int counter;
- int nsys = 1;
- int dcuId[10];
- int ii;
- char *gds_tp_dir = 0;
-
- printf("\n %s compiled %s : %s\n\n",argv[0],__DATE__,__TIME__);
-
- if (argc<3) {
- Usage();
- return(-1);
- }
-
- /* Get the parameters */
- while ((counter = getopt(argc, argv, "r:n:g:l:s:m:h:a:")) != EOF)
- switch(counter) {
-
- case 'r':
- rank = atoi(optarg);
- break;
-
- case 'n':
- nodes = atoi(optarg);
- break;
-
- case 'g':
- segmentId = atoi(optarg);
- break;
-
- case 'l':
- loops = atoi(optarg);
- break;
-
- case 's':
- segmentSize = atoi(optarg);
- if (segmentSize < 4096){
- printf("Min segment size is 4 KB\n");
- return -1;
- }
- break;
-
- case 'm':
- sysname = optarg;
- printf ("sysnames = %s\n",sysname);
- continue;
-
- case 'a':
- localAdapterNo = atoi(optarg);
- continue;
-
- case 'h':
- Usage();
- return(0);
- }
- if(sysname != NULL) {
- printf("System names: %s\n", sysname);
- sprintf(modelnames[0],"%s",strtok(sysname, " "));
- for(;;) {
- char *s = strtok(0, " ");
- if (!s) break;
- sprintf(modelnames[nsys],"%s",s);
- dcuId[nsys] = 0;
- nsys++;
- }
- } else {
- Usage();
- return(0);
- }
-
- for(ii=0;ii<nsys;ii++) {
- char shmem_fname[128];
- sprintf(shmem_fname, "%s_daq", modelnames[ii]);
- void *dcu_addr = findSharedMemory(shmem_fname);
- if (dcu_addr <= 0) {
- fprintf(stderr, "Can't map shmem\n");
- exit(-1);
- } else {
- printf(" %s mapped at 0x%lx\n",modelnames[ii],(unsigned long)dcu_addr);
- }
- shmIpcPtr[ii] = (struct rmIpcStr *)((char *)dcu_addr + CDS_DAQ_NET_IPC_OFFSET);
- shmDataPtr[ii] = ((char *)dcu_addr + CDS_DAQ_NET_DATA_OFFSET);
- shmTpTable[ii] = (struct cdsDaqNetGdsTpNum *)((char *)dcu_addr + CDS_DAQ_NET_GDS_TP_TABLE_OFFSET);
- }
- for (ii = 0; ii < nsys; ii++) {
- modelrates[ii] = getmodelrate(modelnames[ii], gds_tp_dir);
- if (modelrates[ii] == 0) {
- fprintf(stderr, "Unable to determine the rate of %s\n", modelnames[ii]);
- exit(1);
- }
- }
-
- for (;shmIpcPtr[0]->cycle;) usleep(1000);
- int lastCycle = 0;
- printf("Found cycle zero \n");
- int new_cycle;
- // Wait for cycle count update from FE
- do{
- usleep(1000);
- new_cycle = shmIpcPtr[0]->cycle;
- } while (new_cycle == lastCycle);
- printf("New cycle = %d\n", shmIpcPtr[0]->cycle);
- printf("Size of rmIpcStr = %ld\n", sizeof(struct rmIpcStr));
- printf("Size of GDStp = %ld\n", sizeof(struct cdsDaqNetGdsTpNum));
- drIntData = (int *)shmDataPtr[0];
- drIntData += 2;
- printf("CPU = %d\n",*drIntData);
-
- signal(SIGINT,intHandler);
-
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
-
- do {
- error = send_via_reflective_memory(nsys);
- } while (error == SCI_ERR_OK && keepRunning == 1);
-
- error = dolphin_closeout();
- return SCI_ERR_OK;
-}
diff --git a/src/ix_stream/ix_rcvr.c b/src/ix_stream/ix_rcvr.c
deleted file mode 100644
index 00cffcff3..000000000
--- a/src/ix_stream/ix_rcvr.c
+++ /dev/null
@@ -1,348 +0,0 @@
-
-/*********************************************************************************
- * *
- * Copyright (C) 1993 - 2015 *
- * Dolphin Interconnect Solutions AS *
- * *
- * This program is free software; you can redistribute it and/or modify *
- * it under the terms of the GNU General Public License as published by *
- * the Free Software Foundation; either version 2 of the License, *
- * or (at your option) any later version. *
- * *
- * This program is distributed in the hope that it will be useful, *
- * but WITHOUT ANY WARRANTY; without even the implied warranty of *
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
- * GNU General Public License for more details. *
- * *
- * You should have received a copy of the GNU General Public License *
- * along with this program; if not, write to the Free Software *
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
- * *
- * *
- *********************************************************************************/
-
-
-/*********************************************************************************/
-/* */
-/* This program demonstrates the use of the SISCI Reflective Memory */
-/* functionality available with the Dolphin Express DX technology. */
-/* */
-/* This functionality is not available for the SCI technology. */
-/* */
-/*********************************************************************************/
-
-
-#ifdef _WIN32
-#ifndef OS_IS_WINDOWS
-#define OS_IS_WINDOWS 1
-#endif /* !OS_IS_WINDOWS */
-#endif /* _WIN32 */
-
-
-#ifdef OS_IS_WINDOWS
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include "testlib.h"
-#include <malloc.h>
-
-#include "../drv/crc.c"
-#include "../include/daqmap.h"
-#include "../include/drv/fb.h"
-#include "../include/daq_core.h"
-
-
-
-#define __CDECL
-
-#define NO_CALLBACK NULL
-#define NO_FLAGS 0
-#define DATA_TRANSFER_READY 8
-#define CMD_READY 1234
-
-/* Use upper 4 KB of segment for synchronization. */
-#define SYNC_OFFSET ((segmentSize) / 4 - 1024)
-
-#define FILTER 0x12345
-#define MATCH_VAL 0xabcdef
-#define DFLT_EID 1
-#define DFLT_LEN 8192
-#define DFLT_END 128
-#define MAX_LEN (1024*1024*1024)
-#define DFLT_ITER 1000
-#define NUM_RREQ 16 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-#define NUM_SREQ 256 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-
-#define DO_HANDSHAKE 0
-#define MATCH_VAL_MAIN (1 << 31)
-#define MATCH_VAL_THREAD 1
-
-// #define MY_DCU_OFFSET 0x1a00000
-#define MY_DCU_OFFSET 0x00000
-#define MY_IPC_OFFSET (MY_DCU_OFFSET + 0x8000)
-#define MY_GDS_OFFSET (MY_DCU_OFFSET + 0x9000)
-#define MY_DAT_OFFSET (MY_DCU_OFFSET + 0xa000)
-
-#include "./dolphin_common.c"
-extern void *findSharedMemory(char *);
-
-static struct rmIpcStr *shmIpcPtr[128];
-static char *shmDataPtr[128];
-static struct cdsDaqNetGdsTpNum *shmTpTable[128];
-static const int header_size = sizeof(struct daq_fe_header_t);
-static const int buf_size = DAQ_DCU_BLOCK_SIZE;
-char modelnames[DAQ_TRANSIT_MAX_DCU][64];
-char *sysname;
-int modelrates[DAQ_TRANSIT_MAX_DCU];
-daq_multi_dcu_data_t ixDataBlock;
-char *daqbuffer = (char *) &ixDataBlock;
-daq_fe_data_t *zbuffer;
-unsigned int loops = 170;
-static const int ipcSize = sizeof(struct daq_msg_header_t);
-
-
-int signOff(int rank,sci_sequence_t sequence,volatile unsigned int *readAddr,volatile unsigned int *writeAddr)
-{
- /* Lets write CMD_READY the to client, offset "myrank" */
- *(writeAddr+SYNC_OFFSET+rank) = 0;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-}
-
-int waitSender(int rank,sci_sequence_t sequence,volatile unsigned int *readAddr,volatile unsigned int *writeAddr)
-{
-
- int wait_loops = 0;
- int value;
-
-
- /* Lets write CMD_READY the to client, offset "myrank" */
- *(writeAddr+SYNC_OFFSET+rank) = CMD_READY;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-
- printf("Wait for CMD_READY from master ...\n");
-
- /* Lets wait for the client to send me CMD_READY in offset 0 */
- do {
-
- value = (*(readAddr+SYNC_OFFSET));
- wait_loops++;
-
- if ((wait_loops % 100000000)==0) {
- /* Lets again write CMD_READY to the client, offset "myrank" */
- *(writeAddr+SYNC_OFFSET+rank) = CMD_READY;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-
- } while (value != CMD_READY);
- printf("Server received CMD_READY\n");
- *(writeAddr+SYNC_OFFSET+rank) = 0;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-}
-
-
-/*********************************************************************************/
-/* U S A G E */
-/* */
-/*********************************************************************************/
-
-void Usage()
-{
- printf("Usage of reflective:\n");
- printf("reflective -client -nodes <nodes>[ -a <adapter no> -size <segment size> ] \n");
- printf("reflective -server -rank <rank> [ -a <adapter no> -size <segment size> ] \n\n");
- printf(" -client : %s\n", (client) ? "The local node is client" : "The local node is server");
- printf(" -a <value> : Local adapter number (default %d)\n", localAdapterNo);
- printf(" -size <value> : Segment size (default %d)\n", segmentSize);
- printf(" -group <value> : Reflective group identifier (0..5))\n");
- printf(" -rank <value> : Rank of server nodes (1,2,3,4,5,6,7, 8,9)\n");
- printf(" -nodes <value> : Number of servers.\n");
- printf(" -loops <value> : Loops to execute (default %d)\n", loops);
- printf(" -help : This helpscreen\n");
- printf("\n");
-}
-
-
-
-sci_error_t ix_rcv_reflective_memory()
-{
-
- unsigned int value;
- unsigned int written_value = 0;
- double average;
- timer_start_t timer_start;
- int verbose = 1;
- int node_offset;
-
-
-
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- /* Perform a barrier operation. The client acts as master. */
- waitSender(rank,sequence,readAddr,writeAddr);
-
- printf("\n***********************************************************\n\n");
-
- printf("Loops: %d\n", loops);
-
- writeAddr += 256;
- readAddr += 256;
-
- written_value=1;
- int ii;
- int new_cycle = 0;
- int lastCycle = 0;
- int msgSize = 0;
-
- do {
-
- int wait_loops = 0;
-
- /* Lets wait for the client to send me a value in offset 0 */
-
- if (verbose) {
- printf("Wait for broadcast message...\n");
- }
-
- do {
-
- value = (*(readAddr));
- wait_loops++;
- // if ((wait_loops % 10000000)==0) {
- // printf("Value = %d delayed from Client - written_value=%d\n", value,written_value);
- // }
- } while (value != written_value);
-
- printf("zbuff count = %d\n",zbuffer->header.dcuTotalModels);
- printf("zbuff size = %d\n",zbuffer->header.dataBlockSize);
- printf("zbuff dcuid = %d\n",zbuffer->header.dcuheader[0].dcuId);
- printf("zbuff cycle = %d\n",zbuffer->header.dcuheader[0].cycle);
- msgSize = header_size + zbuffer->header.dataBlockSize;
- printf("buff size = %d\t%d\t%d\n",msgSize,header_size,ipcSize);
-
-
- if (verbose) {
- printf("Received broadcast value %d \n",value);
- }
-
- written_value++;
- /* Lets write a value back received value +1 to the client, offset "myrank" */
- *(writeAddr+rank) = written_value;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- } while (written_value < loops); /* do this number of loops */
-
- printf("\n***********************************************************\n\n");
-
- /* Lets clean up after demonstrating the use of reflective memory */
-
- return SCI_ERR_OK;
-}
-
-
-/*********************************************************************************/
-/* M A I N */
-/* */
-/*********************************************************************************/
-
-int __CDECL
-main(int argc,char *argv[])
-{
- int counter;
- volatile unsigned char *daq_read_addr;
-
- printf("\n %s compiled %s : %s\n\n",argv[0],__DATE__,__TIME__);
-
- if (argc<3) {
- Usage();
- return(-1);
- }
-
- /* Get the parameters */
- for (counter=1; counter<argc; counter++) {
-
- if (!strcmp("-rank",argv[counter])) {
- /*LINTED*/
- rank = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-nodes",argv[counter])) {
- /*LINTED*/
- nodes = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-group",argv[counter])) {
- /*LINTED*/
- segmentId = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-loops",argv[counter])) {
- loops = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-size",argv[counter])) {
- segmentSize = strtol(argv[counter+1],(char **) NULL,10);
- if (segmentSize < 4096){
- printf("Min segment size is 4 KB\n");
- return -1;
- }
- continue;
- }
- if (!strcmp("-models",argv[counter])) {
- sysname = argv[counter+1];
- printf ("sysnames = %s\n",sysname);
- continue;
- }
-
- if (!strcmp("-a",argv[counter])) {
- localAdapterNo = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-rn",argv[counter])) {
- continue;
- }
-
- if (!strcmp("-client",argv[counter])) {
- client = 1;
- continue;
- }
-
- if (!strcmp("-server",argv[counter])) {
- server = 1;
- continue;
- }
-
- if (!strcmp("-help",argv[counter])) {
- Usage();
- return(0);
- }
- }
-
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- daq_read_addr = (unsigned char *)readAddr + MY_DAT_OFFSET;
- zbuffer = (daq_fe_data_t *)daq_read_addr;
-
- printf("Calling recvr \n");
-
- error = ix_rcv_reflective_memory();
-
- signOff(rank,sequence,readAddr,writeAddr);
-
- error = dolphin_closeout();
-
- return SCI_ERR_OK;
-}
diff --git a/src/ix_stream/ix_rcvr_threads.c b/src/ix_stream/ix_rcvr_threads.c
deleted file mode 100644
index 906b1f2c7..000000000
--- a/src/ix_stream/ix_rcvr_threads.c
+++ /dev/null
@@ -1,420 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include "testlib.h"
-#include <malloc.h>
-#include <unistd.h>
-#include <pthread.h>
-#include <signal.h>
-#include <time.h>
-
-
-#include "../drv/crc.c"
-#include "../include/daqmap.h"
-#include "../include/drv/fb.h"
-#include "../include/daq_core.h"
-
-
-
-#define __CDECL
-
-
-#define DO_HANDSHAKE 0
-
-// #define MY_DCU_OFFSET 0x1a00000
-#define MY_DCU_OFFSET 0x00000
-#define MY_IPC_OFFSET (MY_DCU_OFFSET + 0x1000)
-#define MY_GDS_OFFSET (MY_DCU_OFFSET + 0x9000)
-#define MY_DAT_OFFSET (MY_DCU_OFFSET + 0xa000)
-
-#include "./dolphin_common.c"
-extern void *findSharedMemorySize(char *,int);
-
-static struct rmIpcStr *shmIpcPtr[128];
-static char *shmDataPtr[128];
-static struct cdsDaqNetGdsTpNum *shmTpTable[128];
-static const int header_size = sizeof(struct daq_fe_header_t);
-static const int buf_size = DAQ_DCU_BLOCK_SIZE;
-char modelnames[DAQ_TRANSIT_MAX_DCU][64];
-char *sysname;
-int modelrates[DAQ_TRANSIT_MAX_DCU];
-daq_multi_dcu_data_t *ixDataBlock;
-static const int mdcu_header_size = sizeof(struct daq_multi_dcu_header_t);
-daq_fe_data_t *zbuffer;
-unsigned int loops = 170;
-static const int ipcSize = sizeof(struct daq_msg_header_t);
-unsigned int tstatus[16];
-int thread_index[DCU_COUNT];
-daq_multi_dcu_data_t mxDataBlockG[32][16];
-int stop_working_threads = 0;
-int start_acq = 0;
-static volatile int keepRunning = 1;
-daq_multi_cycle_data_t *mcd;
-
-
-int waitSender(int rank,sci_sequence_t sequence,volatile unsigned int *readAddr,volatile unsigned int *writeAddr)
-{
-
- int wait_loops = 0;
- int value;
-
-
- /* Lets write CMD_READY the to client, offset "myrank" */
- *(writeAddr+IX_SYNC_OFFSET+rank) = CMD_READY;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-
- printf("Wait for CMD_READY from master ...\n");
-
- /* Lets wait for the client to send me CMD_READY in offset 0 */
- do {
-
- value = (*(readAddr+IX_SYNC_OFFSET));
- wait_loops++;
-
- if ((wait_loops % 100000000)==0) {
- /* Lets again write CMD_READY to the client, offset "myrank" */
- *(writeAddr+IX_SYNC_OFFSET+rank) = CMD_READY;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-
- } while (value != CMD_READY);
- printf("Server received CMD_READY\n");
- *(writeAddr+IX_SYNC_OFFSET+rank) = 0;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-}
-
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-static int64_t
-s_clock (void)
-{
-struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-/*********************************************************************************/
-/* U S A G E */
-/* */
-/*********************************************************************************/
-
-void Usage()
-{
- printf("Usage of reflective:\n");
- printf("reflective -client -nodes <nodes>[ -a <adapter no> -size <segment size> ] \n");
- printf("reflective -server -rank <rank> [ -a <adapter no> -size <segment size> ] \n\n");
- printf(" -client : %s\n", (client) ? "The local node is client" : "The local node is server");
- printf(" -a <value> : Local adapter number (default %d)\n", localAdapterNo);
- printf(" -size <value> : Segment size (default %d)\n", segmentSize);
- printf(" -group <value> : Reflective group identifier (0..5))\n");
- printf(" -rank <value> : Rank of server nodes (1,2,3,4,5,6,7, 8,9)\n");
- printf(" -nodes <value> : Number of servers.\n");
- printf(" -loops <value> : Loops to execute (default %d)\n", loops);
- printf(" -help : This helpscreen\n");
- printf("\n");
-}
-
-
-
-void *ix_rcvr_thread(void *arg)
-{
-
- unsigned int value;
- unsigned int written_value = 0;
- double average;
- int verbose = 1;
- int node_offset;
- int *mythread = (int *)arg;
- int mt = *mythread;
- int ii;
- int cycle = 0;
- int lastCycle = 0;
- int msgSize = 0;
- volatile unsigned char *daq_read_addr;
- daq_fe_data_t *ixbuffer;
- volatile unsigned int *myreadAddr;
- volatile unsigned int *mywriteAddr;
-
- myreadAddr = (unsigned int *)readAddr;
- mywriteAddr = (unsigned int *)writeAddr;
-
- daq_read_addr = (unsigned char *)readAddr + MY_DAT_OFFSET;
- ixbuffer = (daq_fe_data_t *)daq_read_addr;
- ixbuffer += mt;
-
- printf("Read = 0x%lx \n",(long)ixbuffer);
-
- /* Perform a barrier operation. The client acts as master. */
- waitSender(rank,sequence,readAddr,writeAddr);
-
- printf("\n***********************************************************\n\n");
-
- printf("Loops: %d\n", loops);
-
- mywriteAddr += 256;
- myreadAddr += 256;
-
- written_value=1;
-
- do {
-
- int wait_loops = 0;
-
- /* Lets wait for the client to send me a value in offset 0 */
-
- if (!verbose) {
- printf("Wait for broadcast message...\n");
- }
-
- do {
-
- value = (*(myreadAddr));
- wait_loops++;
- // if ((wait_loops % 10000000)==0) {
- // printf("Value = %d delayed from Client - written_value=%d\n", value,written_value);
- // }
- } while (value != written_value);
-
- cycle = ixbuffer->header.dcuheader[0].cycle;
- msgSize = header_size + ixbuffer->header.dataBlockSize;
- char *localbuff = (char *)&mxDataBlockG[mt][cycle];
- memcpy(localbuff,ixbuffer,msgSize);
-
-#if 0
-
- printf("zbuff count = %d\n",ixbuffer->header.dcuTotalModels);
- printf("zbuff size = %d\n",ixbuffer->header.dataBlockSize);
- printf("zbuff cycle = %d\n",cycle);
- printf("zbuff dcuid = %d\n",ixbuffer->header.dcuheader[0].dcuId);
- printf("buff size = %d\t%d\t%d\n",msgSize,header_size,ipcSize);
-
-#endif
- if (!verbose) {
- printf("Received broadcast value %d \n",cycle);
- }
- tstatus[cycle] |= (1 << mt);
-
- written_value++;
- /* Lets write a value back received value +1 to the client, offset "myrank" */
- *(mywriteAddr+rank) = written_value;
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
-
-
- } while (!stop_working_threads); /* do this number of loops */
-
- printf("Stopping thread %d\n",mt);
- usleep(200000);
-
- return 0;
-}
-
-
-/*********************************************************************************/
-/* M A I N */
-/* */
-/*********************************************************************************/
-
-int __CDECL
-main(int argc,char *argv[])
-{
- int counter;
- volatile unsigned char *daq_read_addr;
- pthread_t thread_id[4];
- int ii,jj;
- int dataRdy = 0;
- int loop = 0;
- daq_msg_header_t *sendheader;
- daq_msg_header_t *rcvheader;
- char *senddata;
- char *rcvdata;
- int myc;
- char *mcdDataPtr;
-
- printf("\n %s compiled %s : %s\n\n",argv[0],__DATE__,__TIME__);
- printf("Size of mcd = %d\n",sizeof(mcd));
-
- if (argc<3) {
- Usage();
- return(-1);
- }
-
- /* Get the parameters */
- for (counter=1; counter<argc; counter++) {
-
- if (!strcmp("-rank",argv[counter])) {
- /*LINTED*/
- rank = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-nodes",argv[counter])) {
- /*LINTED*/
- nodes = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-group",argv[counter])) {
- /*LINTED*/
- segmentId = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-size",argv[counter])) {
- segmentSize = strtol(argv[counter+1],(char **) NULL,10);
- if (segmentSize < 4096){
- printf("Min segment size is 4 KB\n");
- return -1;
- }
- continue;
- }
-
- if (!strcmp("-a",argv[counter])) {
- localAdapterNo = strtol(argv[counter+1],(char **) NULL,10);
- continue;
- }
-
- if (!strcmp("-help",argv[counter])) {
- Usage();
- return(0);
- }
- }
-
- signal(SIGINT,intHandler);
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- char *ifo = (char *)findSharedMemorySize("ifo",100);
- daq_multi_cycle_header_t *ifo_header = (daq_multi_cycle_header_t *)ifo;
- char *ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
-
- printf("Starting recvr threads\n");
- for(ii=0;ii<nodes;ii++)
- {
- thread_index[ii] = ii;
- pthread_create(&thread_id[ii],NULL,ix_rcvr_thread,(void *)&thread_index);
- dataRdy |= (1 << ii);
- }
-
-
- int timeout = 0;
- int resync = 1;
- int64_t mytime = 0;
- int64_t mylasttime = 0;
- int64_t myptime = 0;
- int mytotaldcu = 0;
-
- do {
- if(resync) {
- loop = 0;
- do {
- usleep(2000);
- timeout += 1;
- }while(tstatus[loop] == 0 && timeout < 5000);
- for(ii=0;ii<16;ii++) tstatus[ii] = 0;
- printf("RESYNC ***************** \n");
- }
- // Wait until received data from at least 1 FE
- timeout = 0;
- do {
- usleep(2000);
- timeout += 1;
- }while(tstatus[loop] == 0 && timeout < 5000 && !resync);
- resync = 0;
- // If timeout, not getting data from anyone.
- if(timeout >= 5000) resync = 1;
- if (resync) continue;
-
- // Wait until data received from everyone
- timeout = 0;
- do {
- usleep(1000);
- timeout += 1;
- }while(tstatus[loop] != dataRdy && timeout < 5);
- // If timeout, not getting data from everyone.
- // TODO: MARK MISSING FE DATA AS BAD
-
- // Clear thread rdy for this cycle
- tstatus[loop] = 0;
-
- // Timing diagnostics
- mytime = s_clock();
- myptime = mytime - mylasttime;
- mylasttime = mytime;
- if(loop == 0) {
- printf("Data rdy for cycle = %d\t%ld\n",mxDataBlockG[0][loop].header.dcuheader[0].timeSec,myptime);
- printf("\tdatasize = %d\n",mxDataBlockG[0][loop].header.dataBlockSize);
- }
- // Reset total DCU counter
- mytotaldcu = 0;
- // daq_multi_cycle_header_t *ifo_header = (daq_multi_cycle_header_t header *)ifo;
- // char *ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
- mcdDataPtr = ifo_data;
- mcdDataPtr += loop * DAQ_TRANSIT_DC_DATA_BLOCK_SIZE;
- ixDataBlock = (daq_multi_dcu_data_t *)mcdDataPtr;
- sendheader = (daq_msg_header_t *) &ixDataBlock->header.dcuheader[0];
- senddata = (char *) &ixDataBlock->dataBlock[0];
- int sendDataBlockSize = 0;
- // Loop over all data buffers received from FE computers
- for(ii=0;ii<nodes;ii++) {
- // for(jj=0;jj<2;jj++) {
- // Get pointers to receiver header and data blocks
- rcvheader = (daq_msg_header_t *) &mxDataBlockG[ii][loop].header.dcuheader[0];
- rcvdata = (char *) &mxDataBlockG[ii][loop].dataBlock[0];
- // Get the receive number of dcu
- myc = mxDataBlockG[ii][loop].header.dcuTotalModels;
- // Add rcv dcu count to my total send count
- mytotaldcu += myc;
- // Calc size of receive header data to move
- int headersize = myc * ipcSize;
- // Copy rcv header area to send header
- memcpy(sendheader,rcvheader,headersize);
- // Increment send header for next dcu
- sendheader += myc;
- // Get the size of received data to move
- sendDataBlockSize += mxDataBlockG[ii][loop].header.dataBlockSize;
- // Copy rcv data to send data block
- memcpy(senddata,rcvdata,sendDataBlockSize);
- // Update the send header with dcu and size info
- ixDataBlock->header.dcuTotalModels = mytotaldcu;
- ixDataBlock->header.dataBlockSize = sendDataBlockSize;
- // }
- }
- ifo_header->curCycle = loop;
- ifo_header->cycleDataSize = DAQ_TRANSIT_DC_DATA_BLOCK_SIZE;
- ifo_header->maxCycle = 16;
-
-#if 0
- if(loop == 0) {
- printf("New Send Data:\n");
- printf("\tNum of DCU: %d\n",ixDataBlock->header.dcuTotalModels);
- printf("\tDataSize: %d\n",ixDataBlock->header.dataBlockSize);
- for(ii=0;ii<ixDataBlock->header.dcuTotalModels;ii++)
- {
- printf("\tDCUID %d = %d\n",ii,ixDataBlock->header.dcuheader[ii].cycle);
- }
- }
-#endif
-
- loop ++;
- loop %= 16;
- }while (keepRunning);
-
- printf("stopping threads %d \n",nodes);
- stop_working_threads = 1;
-
- // Wait for threads to stop
- sleep(2);
-
- error = dolphin_closeout();
-
- return SCI_ERR_OK;
-}
diff --git a/src/ix_stream/mx2ix.c b/src/ix_stream/mx2ix.c
deleted file mode 100644
index 0f903eb2f..000000000
--- a/src/ix_stream/mx2ix.c
+++ /dev/null
@@ -1,829 +0,0 @@
-//
-/// @file mx2ix.c
-/// @brief DAQ data concentrator code. Receives data via Open-MX and sends via Dolphin IX..
-//
-
-#include "myriexpress.h"
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/file.h>
-#include <stdio.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include "../drv/crc.c"
-#include <time.h>
-#include "../include/daqmap.h"
-#include "../include/daq_core.h"
-#include "dc_utils.h"
-
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include "testlib.h"
-
-#include "simple_pv.h"
-
-#define __CDECL
-
-#define DO_HANDSHAKE 0
-
-
-#include "./dolphin_common.c"
-
-#define MIN_DELAY_MS 5
-#define MAX_DELAY_MS 40
-
-#define MAX_FE_COMPUTERS 32
-
-#define MX_MUTEX_T pthread_mutex_t
-#define MX_MUTEX_INIT(mutex_) pthread_mutex_init(mutex_, 0)
-#define MX_MUTEX_LOCK(mutex_) pthread_mutex_lock(mutex_)
-#define MX_MUTEX_UNLOCK(mutex_) pthread_mutex_unlock(mutex_)
-
-#define MX_THREAD_T pthread_t
-#define MX_THREAD_CREATE(thread_, start_routine_, arg_) \
-pthread_create(thread_, 0, start_routine_, arg_)
-#define MX_THREAD_JOIN(thread_) pthread_join(thread, 0)
-
-MX_MUTEX_T stream_mutex;
-
-#define FILTER 0x12345
-#define MATCH_VAL 0xabcdef
-#define DFLT_EID 1
-#define DFLT_LEN 8192
-#define DFLT_END 128
-#define MAX_LEN (1024*1024*1024)
-#define DFLT_ITER 1000
-#define NUM_RREQ 16 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-#define NUM_SREQ 256 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-
-#define DO_HANDSHAKE 0
-#define MATCH_VAL_MAIN (1 << 31)
-#define MATCH_VAL_THREAD 1
-#define THREADS_PER_NIC 16
-#define IX_STOP_SEC 5
-static int xmitDataOffset[IX_BLOCK_COUNT];
-daq_multi_cycle_header_t *xmitHeader[IX_BLOCK_COUNT];
-
-extern void *findSharedMemorySize(char *,int);
-
-int do_verbose = 0;
-
-struct thread_info {
- int index;
- uint32_t match_val;
- uint32_t tpn;
- uint32_t bid;
-};
-struct thread_mon_info {
- int index;
- void *ctx;
-};
-struct thread_info thread_index[DCU_COUNT];
-char *sname[DCU_COUNT]; // Names of FE computers serving DAQ data
-char *local_iface[MAX_FE_COMPUTERS];
-daq_multi_dcu_data_t mxDataBlockSingle[MAX_FE_COMPUTERS];
-int64_t dataRecvTime[MAX_FE_COMPUTERS];
-const int mc_header_size = sizeof(daq_multi_cycle_header_t);
-int stop_working_threads = 0;
-int start_acq = 0;
-static volatile int keepRunning = 1;
-int thread_cycle[MAX_FE_COMPUTERS];
-int thread_timestamp[MAX_FE_COMPUTERS];
-int rcv_errors = 0;
-int dataRdy[MAX_FE_COMPUTERS];
-
-void
-usage()
-{
- fprintf(stderr, "Usage: mx2ix [args] -s server names -m shared memory size -g IX channel \n");
- fprintf(stderr, "-l filename - log file name\n");
- fprintf(stderr, "-s - number of FE computers to connect (1-32): Default = 32\n");
- fprintf(stderr, "-v - verbose prints diag test data\n");
- fprintf(stderr, "-g - Dolphin IX channel to xmit on (0-3)\n");
- fprintf(stderr, "-p - Debug pv prefix, requires -P as well\n");
- fprintf(stderr, "-P - Path to a named pipe to send PV debug information to\n");
- fprintf(stderr, "-d - Max delay in milli seconds to wait for a FE to send data, defaults to 10\n");
- fprintf(stderr, "-t - Number of rcvr threads per NIC: default = 16\n");
- fprintf(stderr, "-n - Data Concentrator number (0 or 1) : default = 0\n");
- fprintf(stderr, "-h - help\n");
-}
-
-// *************************************************************************
-// Timing Diagnostic Routine
-// *************************************************************************
-static int64_t
-s_clock (void)
-{
-struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-// *************************************************************************
-// Catch Control C to end cod in controlled manner
-// *************************************************************************
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-void sigpipeHandler(int dummy)
-{}
-
-// **********************************************************************************************
-void print_diags(int nsys, int lastCycle, int sendLength, daq_multi_dcu_data_t *ixDataBlock,int dbs[]) {
-// **********************************************************************************************
- int ii = 0;
- // Print diags in verbose mode
- fprintf(stderr,"Receive errors = %d\n",rcv_errors);
- fprintf(stderr,"Time = %d\t size = %d\n",ixDataBlock->header.dcuheader[0].timeSec,sendLength);
- fprintf(stderr,"DCU ID\tCycle \t TimeSec\tTimeNSec\tDataSize\tTPCount\tTPSize\tXmitSize\n");
- for(ii=0;ii<nsys;ii++) {
- fprintf(stderr,"%d",ixDataBlock->header.dcuheader[ii].dcuId);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].cycle);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].timeSec);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].timeNSec);
- fprintf(stderr,"\t\t%d",ixDataBlock->header.dcuheader[ii].dataBlockSize);
- fprintf(stderr,"\t\t%d",ixDataBlock->header.dcuheader[ii].tpCount);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].tpBlockSize);
- fprintf(stderr,"\t%d",dbs[ii]);
- fprintf(stderr,"\n ");
- }
-}
-
-
-// *************************************************************************
-// Thread for receiving DAQ data via Open-MX
-// *************************************************************************
-void *rcvr_thread(void *arg) {
- struct thread_info* my_info = (struct thread_info*)arg;
- int mt = my_info->index;
- uint32_t mv = my_info->match_val;
- uint32_t board_id = my_info->bid;
- uint32_t tpn = my_info->tpn;
- int cycle = 0;
- daq_multi_dcu_data_t *mxDataBlock;
- mx_return_t ret;
- int count, len, cur_req;
- mx_status_t stat;
- mx_request_t req[NUM_RREQ];
- mx_segment_t seg;
- uint32_t result;
- char *buffer;
- uint32_t filter;
- mx_endpoint_t ep;
- int myErrorStat = 0;
-
- filter = FILTER;
- int copySize;
-
-
- int dblock = board_id * tpn + mt;
- fprintf(stderr,"Starting receive loop for thread %d %d\n", board_id,mt);
-
- ret = mx_open_endpoint(board_id, mt, filter, NULL, 0, &ep);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to open endpoint %s\n", mx_strerror(ret));
- return(0);
- }
-
- fprintf(stderr,"waiting for someone to connect on CH %d\n",mt);
- len = 0xf00000;
- fprintf(stderr,"buffer length = %d\n",len);
- buffer = (char *)malloc(len);
- if (buffer == NULL) {
- fprintf(stderr, "Can't allocate buffers here\n");
- mx_close_endpoint(ep);
- return(0);
- }
- len /= NUM_RREQ;
- fprintf(stderr," length = %d\n",len);
-
- for (cur_req = 0; cur_req < NUM_RREQ; cur_req++) {
- seg.segment_ptr = &buffer[cur_req * len];
- seg.segment_length = len;
- mx_irecv(ep, &seg, 1, mv, MX_MATCH_MASK_NONE, 0,
- &req[cur_req]);
- }
-
- mx_set_error_handler(MX_ERRORS_RETURN);
-
- char *daqbuffer = (char *)&mxDataBlockSingle[dblock];
- do {
- for (count = 0; count < NUM_RREQ; count++) {
- cur_req = count & (16 - 1);
- mx_wait(ep, &req[cur_req],150, &stat, &result);
- myErrorStat = 0;
- if (!result) {
- myErrorStat = 1;
- count --;
- }
- if (stat.code != MX_STATUS_SUCCESS) {
- myErrorStat = 2;
- }
- if(!myErrorStat) {
- seg.segment_ptr = &buffer[cur_req * len];
- mxDataBlock = (daq_multi_dcu_data_t *)seg.segment_ptr;
- copySize = stat.xfer_length;
- memcpy(daqbuffer,seg.segment_ptr,copySize);
- // Get the message DAQ cycle number
- cycle = mxDataBlock->header.dcuheader[0].cycle;
- // Pass cycle and timestamp data back to main process
- thread_cycle[dblock] = cycle;
- thread_timestamp[dblock] = mxDataBlock->header.dcuheader[0].timeSec;
- dataRecvTime[dblock] = s_clock();
- mx_irecv(ep, &seg, 1, mv, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- }
-
- }
- // Run until told to stop by main thread
- } while(!stop_working_threads);
- fprintf(stderr,"Stopping thread %d\n",mt);
- usleep(200000);
-
- mx_close_endpoint(ep);
- return(0);
-
-}
-
-// *************************************************************************
-// Main Process
-// *************************************************************************
-int
-main(int argc, char **argv)
-{
- pthread_t thread_id[MAX_FE_COMPUTERS];
- unsigned int nsys = MAX_FE_COMPUTERS; // The number of mapped shared memories (number of data sources)
- char *buffer_name = "ifo";
- int c;
- int ii; // Loop counter
- int delay_ms = 10;
- int delay_cycles = 0;
-
- extern char *optarg; // Needed to get arguments to program
-
- // PV/debug information
- char *pv_prefix = 0;
- char *pv_debug_pipe_name = 0;
- int pv_debug_pipe = -1;
-
- // Declare shared memory data variables
- daq_multi_cycle_header_t *ifo_header;
- char *ifo;
- char *ifo_data;
- int cycle_data_size;
- daq_multi_dcu_data_t *ifoDataBlock;
- char *nextData;
- int max_data_size_mb = 100;
- int max_data_size = 0;
- char *mywriteaddr;
- int xmitBlockNum = 0;
- int dc_number = 1;
-
-
- /* set up defaults */
- int xmitData = 0;
- int tpn = THREADS_PER_NIC;
-
-
-
- // Get arguments sent to process
- while ((c = getopt(argc, argv, "b:hs:m:g:vp:P:d:l:t:n:")) != EOF) switch(c) {
- case 's':
- nsys = atoi(optarg);
- if(nsys > MAX_FE_COMPUTERS) {
- fprintf(stderr,"Max number of FE computers is 32\n");
- usage();
- exit(1);
- }
- break;
- case 'v':
- do_verbose = 1;
- break;
- case 'n':
- dc_number = atoi(optarg);
- dc_number ++;
- if(dc_number > 2) {
- fprintf(stderr,"DC number must be 0 or 1\n");
- usage();
- exit(1);
- }
- break;
- case 'm':
- max_data_size_mb = atoi(optarg);
- if (max_data_size_mb < 20){
- fprintf(stderr,"Min data block size is 20 MB\n");
- return -1;
- }
- if (max_data_size_mb > 100){
- fprintf(stderr,"Max data block size is 100 MB\n");
- return -1;
- }
- break;
- case 'g':
- segmentId = atoi(optarg);
- xmitData = 1;
- break;
- case 't':
- tpn = atoi(optarg);
- break;
- case 'b':
- buffer_name = optarg;
- break;
- case 'p':
- pv_prefix = optarg;
- break;
- case 'P':
- pv_debug_pipe_name = optarg;
- break;
- case 'd':
- delay_ms = atoi(optarg);
- if (delay_ms < MIN_DELAY_MS || delay_ms > MAX_DELAY_MS) {
- fprintf(stderr,"The delay factor must be between 5ms and 40ms\n");
- return -1;
- }
- break;
- case 'l':
- if (0 == freopen(optarg, "w", stdout)) {
- perror ("freopen");
- exit (1);
- }
- setvbuf(stdout, NULL, _IOLBF, 0);
- stderr = stdout;
- break;
- case 'h':
- default:
- usage();
- exit(1);
- }
- max_data_size = max_data_size_mb * 1024*1024;
- delay_cycles = delay_ms * 10;
-
- mx_init();
- // MX_MUTEX_INIT(&stream_mutex);
-
- // set up to catch Control C
- signal(SIGINT,intHandler);
- // setup to ignore sig pipe
- signal(SIGPIPE, sigpipeHandler);
-
- fprintf(stderr,"Num of sys = %d\n",nsys);
-
- // Get pointers to local DAQ mbuf
- ifo = (char *)findSharedMemorySize(buffer_name,max_data_size_mb);
- ifo_header = (daq_multi_cycle_header_t *)ifo;
- ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
- cycle_data_size = (max_data_size - sizeof(daq_multi_cycle_header_t)) / DAQ_NUM_DATA_BLOCKS_PER_SECOND;
- cycle_data_size -= (cycle_data_size % 8);
- fprintf (stderr,"cycle data size = %d\t%d\n",cycle_data_size, max_data_size_mb);
- sleep(3);
- ifo_header->cycleDataSize = cycle_data_size;
- ifo_header->maxCycle = DAQ_NUM_DATA_BLOCKS_PER_SECOND;
-
-#if 0
- if(xmitData) {
- // Connect to Dolphin
- error = dolphin_init();
- fprintf(stderr,"Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- // Set pointer to xmit header in Dolphin xmit data area.
- mywriteaddr = (char *)writeAddr;
- for(ii=0;ii<IX_BLOCK_COUNT;ii++) {
- xmitHeader[ii] = (daq_multi_cycle_header_t *)mywriteaddr;
- mywriteaddr += IX_BLOCK_SIZE;
- xmitDataOffset[ii] = IX_BLOCK_SIZE * ii + sizeof(struct daq_multi_cycle_header_t);
- fprintf(stderr,"Dolphin at 0x%lx and 0x%lx",(long)xmitHeader[ii],(long)xmitDataOffset[ii]);
- }
- }
-#endif
-
- fprintf(stderr,"nsys = %d\n",nsys);
-
- // Make 0MQ socket connections
- for(ii=0;ii<nsys;ii++) {
- // Create a thread to receive data from each data server
- thread_index[ii].index = ii % tpn;
- thread_index[ii].match_val = MATCH_VAL_MAIN;
- thread_index[ii].tpn = tpn;
- thread_index[ii].bid = ii/tpn;
- pthread_create(&thread_id[ii],NULL,rcvr_thread,(void *)&thread_index[ii]);
-
- }
- int nextCycle = 0;
- start_acq = 1;
- int64_t mytime = 0;
- int64_t mylasttime = 0;
- int64_t myptime = 0;
- int64_t n_cycle_time = 0;
- int mytotaldcu = 0;
- char *zbuffer;
- size_t zbuffer_remaining = 0;
- int dc_datablock_size = 0;
- int datablock_size_running = 0;
- int datablock_size_mb_s = 0;
- static const int header_size = sizeof(daq_multi_dcu_header_t);
- char dcstatus[4096];
- char dcs[48];
- int edcuid[10];
- int estatus[10];
- int edbs[10];
- unsigned long ets = 0;
- int timeout = 0;
- int threads_rdy;
- int any_rdy = 0;
- int jj,kk;
- int sendLength = 0;
-
- int min_cycle_time = 1 << 30;
- int pv_min_cycle_time = 0;
- int max_cycle_time = 0;
- int pv_max_cycle_time = 0;
- int mean_cycle_time = 0;
- int pv_mean_cycle_time = 0;
- int pv_dcu_count = 0;
- int pv_total_datablock_size = 0;
- int pv_datablock_size_mb_s = 0;
- int uptime = 0;
- int pv_uptime = 0;
- int gps_time = 0;
- int pv_gps_time = 0;
- int missed_flag = 0;
- int missed_nsys[MAX_FE_COMPUTERS];
- int64_t recv_time[MAX_FE_COMPUTERS];
- int64_t min_recv_time = 0;
- int64_t cur_ref_time = 0;
- int recv_buckets[(MAX_DELAY_MS/5)+2];
- int festatus = 0;
- int pv_festatus = 0;
- int ix_xmit_stop = 0;
- SimplePV pvs[] = {
- {
- "RECV_MIN_MS",
- SIMPLE_PV_INT,
- &pv_min_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MAX_MS",
- SIMPLE_PV_INT,
- &pv_max_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MEAN_MS",
- SIMPLE_PV_INT,
- &pv_mean_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "DCU_COUNT",
- SIMPLE_PV_INT,
- &pv_dcu_count,
-
- 120,
- 0,
- 115,
- 0,
- },
- {
- "DATA_SIZE",
- SIMPLE_PV_INT,
- &pv_total_datablock_size,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1*1024*1024,
- },
- {
- "DATA_RATE",
- SIMPLE_PV_INT,
- &pv_datablock_size_mb_s,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1000000,
- },
- {
- "UPTIME_SECONDS",
- SIMPLE_PV_INT,
- &pv_uptime,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
-
- 1,
- },
- {
- "RCV_STATUS",
- SIMPLE_PV_INT,
- &pv_festatus,
-
- 0xffffffff,
- 0,
- 0xfffffffe,
-
- 1,
- },
- {
- "GPS",
- SIMPLE_PV_INT,
- &pv_gps_time,
-
- 0xfffffff,
- 0,
- 0xfffffffe,
-
- 1,
- },
-
- };
- if (pv_debug_pipe_name)
- {
- pv_debug_pipe = open(pv_debug_pipe_name, O_NONBLOCK | O_RDWR, 0);
- if (pv_debug_pipe < 0) {
- fprintf(stderr, "Unable to open %s for writting (pv status)\n", pv_debug_pipe_name);
- exit(1);
- }
- }
-
- missed_flag = 1;
- memset(&missed_nsys[0], 0, sizeof(missed_nsys));
- memset(recv_buckets, 0, sizeof(recv_buckets));
- do {
- // Reset counters
- timeout = 0;
- for(ii=0;ii<nsys;ii++) dataRdy[ii] = 0;
- for(ii=0;ii<nsys;ii++) thread_cycle[ii] = 50;
- threads_rdy = 0;
- any_rdy = 0;
-
- // Wait up to 100ms until received data from at least 1 FE or timeout
- do {
- usleep(2000);
- for(ii=0;ii<nsys;ii++) {
- if(nextCycle == thread_cycle[ii]) any_rdy = 1;
- }
- timeout += 1;
- }while(!any_rdy && timeout < 50);
-
- // Wait up to delay_ms ms in 1/10ms intervals until data received from everyone or timeout
- timeout = 0;
- do {
- usleep(100);
- for(ii=0;ii<nsys;ii++) {
- if(nextCycle == thread_cycle[ii] && !dataRdy[ii]) threads_rdy ++;
- if(nextCycle == thread_cycle[ii]) dataRdy[ii] = 1;
- }
- timeout += 1;
- }while(threads_rdy < nsys && timeout < delay_cycles);
- if(timeout >= 100) rcv_errors += (nsys - threads_rdy);
-
- if(any_rdy) {
- int tbsize = 0;
- // Timing diagnostics
- mytime = s_clock();
- myptime = mytime - mylasttime;
- mylasttime = mytime;
- if (myptime < min_cycle_time) {
- min_cycle_time = myptime;
- }
- if (myptime > max_cycle_time) {
- max_cycle_time = myptime;
- }
- mean_cycle_time += myptime;
- ++n_cycle_time;
-
- // Reset total DCU counter
- mytotaldcu = 0;
- // Reset total DC data size counter
- dc_datablock_size = 0;
- // Get pointer to next data block in shared memory
- nextData = (char *)ifo_data;
- nextData += cycle_data_size * nextCycle;
- ifoDataBlock = (daq_multi_dcu_data_t *)nextData;
- zbuffer = (char *)nextData + header_size;
- zbuffer_remaining = cycle_data_size - header_size;
-
- min_recv_time = 0x7fffffffffffffff;
- festatus = 0;
- // Loop over all data buffers received from FE computers
- for(ii=0;ii<nsys;ii++) {
- recv_time[ii] = -1;
- if(dataRdy[ii]) {
- festatus += (1 << ii);
- recv_time[ii] = dataRecvTime[ii];
- if (recv_time[ii] < min_recv_time) {
- min_recv_time = recv_time[ii];
- }
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "+++%d\n", ii);
- }
- int myc = mxDataBlockSingle[ii].header.dcuTotalModels;
- // For each model, copy over data header information
- for(jj=0;jj<myc;jj++) {
- // Copy data header information
- ifoDataBlock->header.dcuheader[mytotaldcu].dcuId =
- mxDataBlockSingle[ii].header.dcuheader[jj].dcuId;
- ifoDataBlock->header.dcuheader[mytotaldcu].fileCrc =
- mxDataBlockSingle[ii].header.dcuheader[jj].fileCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].status =
- mxDataBlockSingle[ii].header.dcuheader[jj].status;
- ifoDataBlock->header.dcuheader[mytotaldcu].cycle =
- mxDataBlockSingle[ii].header.dcuheader[jj].cycle;
- if(!ix_xmit_stop) {
- ifoDataBlock->header.dcuheader[mytotaldcu].timeSec =
- mxDataBlockSingle[ii].header.dcuheader[jj].timeSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeNSec =
- mxDataBlockSingle[ii].header.dcuheader[jj].timeNSec;
- }
- ifoDataBlock->header.dcuheader[mytotaldcu].dataCrc =
- mxDataBlockSingle[ii].header.dcuheader[jj].dataCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataBlockSize =
- mxDataBlockSingle[ii].header.dcuheader[jj].dataBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpBlockSize =
- mxDataBlockSingle[ii].header.dcuheader[jj].tpBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount =
- mxDataBlockSingle[ii].header.dcuheader[jj].tpCount;
-
- // Check for downstream DAQ reset request from EDCU
- if(mxDataBlockSingle[ii].header.dcuheader[jj].dcuId == 52 &&
- mxDataBlockSingle[ii].header.dcuheader[jj].tpCount == dc_number)
- {
- fprintf(stderr,"Got a DAQ Reset REQUEST %u\n",mxDataBlockSingle[ii].header.dcuheader[jj].timeSec);
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount = 0;
- ix_xmit_stop = 16 * IX_STOP_SEC;
- }
- for(kk=0;kk<DAQ_GDS_MAX_TP_NUM ;kk++)
- ifoDataBlock->header.dcuheader[mytotaldcu].tpNum[kk] = mxDataBlockSingle[ii].header.dcuheader[jj].tpNum[kk];
- edbs[mytotaldcu] = mxDataBlockSingle[ii].header.dcuheader[jj].tpBlockSize + mxDataBlockSingle[ii].header.dcuheader[jj].dataBlockSize;
- // Get some diags
- if(ifoDataBlock->header.dcuheader[mytotaldcu].status != 0xbad)
- ets = mxDataBlockSingle[ii].header.dcuheader[jj].timeSec;
- estatus[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].status;
- edcuid[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].dcuId;
- // Increment total DCU count
- mytotaldcu ++;
- }
- // Get the size of the data to transfer
- int mydbs = mxDataBlockSingle[ii].header.fullDataBlockSize;
- // Get pointer to data in receive data block
- char *mbuffer = (char *)&mxDataBlockSingle[ii].dataBlock[0];
- if (mydbs > zbuffer_remaining)
- {
- fprintf(stderr,"Buffer overflow found. Attempting to write %d bytes to zbuffer which has %d bytes remainging\n",
- (int)mydbs, (int)zbuffer_remaining);
- abort();
- }
- // Copy data from receive buffer to shared memory
- memcpy(zbuffer,mbuffer,mydbs);
- // Increment shared memory data buffer pointer for next data set
- zbuffer += mydbs;
- // Calc total size of data block for this cycle
- dc_datablock_size += mydbs;
- tbsize += mydbs;
- } else {
- missed_nsys[ii] |= missed_flag;
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "---%d\n", ii);
- }
- }
- }
- // Write total data block size to shared memory header
- ifoDataBlock->header.fullDataBlockSize = dc_datablock_size;
- // Write total dcu count to shared memory header
- ifoDataBlock->header.dcuTotalModels = mytotaldcu;
- // Set multi_cycle head cycle to indicate data ready for this cycle
- ifo_header->curCycle = nextCycle;
- xmitBlockNum = nextCycle % IX_BLOCK_COUNT;
-
- // Calc IX message size
- sendLength = header_size + ifoDataBlock->header.fullDataBlockSize;
- for (ii = 0; ii < nsys; ++ii) {
- cur_ref_time = 0;
- recv_time[ii] -= min_recv_time;
- }
- datablock_size_running += dc_datablock_size;
- if (nextCycle == 0) {
- datablock_size_mb_s = datablock_size_running / 1024 ;
- pv_datablock_size_mb_s = datablock_size_mb_s;
- uptime ++;
- pv_uptime = uptime;
- gps_time = ifoDataBlock->header.dcuheader[0].timeSec;
- pv_gps_time = gps_time;
- pv_dcu_count = mytotaldcu;
- pv_festatus = festatus;
- mean_cycle_time = (n_cycle_time > 0 ? mean_cycle_time / n_cycle_time : 1 << 31);
-
- pv_mean_cycle_time = mean_cycle_time;
- pv_max_cycle_time = max_cycle_time;
- pv_min_cycle_time = min_cycle_time;
- send_pv_update(pv_debug_pipe, pv_prefix, pvs, sizeof(pvs)/sizeof(pvs[0]));
-
- if (do_verbose) {
- fprintf(stderr,"\nData rdy for cycle = %d\t\tTime Interval = %ld msec\n", nextCycle, myptime);
- fprintf(stderr,"Min/Max/Mean cylce time %d/%d/%d msec over %ld cycles\n", min_cycle_time, max_cycle_time,
- mean_cycle_time, n_cycle_time);
- fprintf(stderr,"Total DCU = %d\t\t\tBlockSize = %d\n", mytotaldcu, dc_datablock_size);
- print_diags(mytotaldcu, nextCycle, sendLength, ifoDataBlock, edbs);
- }
- n_cycle_time = 0;
- min_cycle_time = 1 << 30;
- max_cycle_time = 0;
- mean_cycle_time = 0;
-
-
- missed_flag = 1;
- datablock_size_running = 0;
- } else {
- missed_flag <<= 1;
- }
-#if 0
- if(xmitData && !ix_xmit_stop) {
- if (sendLength > IX_BLOCK_SIZE)
- {
- fprintf(stderr, "Buffer overflow. Sending %d bytes into a dolphin block that holds %d\n",
- (int)sendLength, (int)IX_BLOCK_SIZE);
- abort();
- }
- // WRITEDATA to Dolphin Network
- SCIMemCpy(sequence,nextData, remoteMap,xmitDataOffset[xmitBlockNum],sendLength,memcpyFlag,&error);
- error = SCI_ERR_OK;
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- fprintf(stderr,"For reference the expected error codes are:\n");
- fprintf(stderr,"SCI_ERR_OUT_OF_RANGE = 0x%x\n", SCI_ERR_OUT_OF_RANGE);
- fprintf(stderr,"SCI_ERR_SIZE_ALIGNMENT = 0x%x\n", SCI_ERR_SIZE_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_OFFSET_ALIGNMENT = 0x%x\n", SCI_ERR_OFFSET_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_TRANSFER_FAILED = 0x%x\n", SCI_ERR_TRANSFER_FAILED);
- return error;
- }
- // Set data header information
- unsigned int maxCycle = ifo_header->maxCycle;
- unsigned int curCycle = ifo_header->curCycle;
- xmitHeader[xmitBlockNum]->maxCycle = maxCycle;
- xmitHeader[xmitBlockNum]->cycleDataSize = sendLength;;
- // Send cycle last as indication of data ready for receivers
- xmitHeader[xmitBlockNum]->curCycle = curCycle;
- // Have to flush the buffers to make data go onto Dolphin network
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-#endif
- if(ix_xmit_stop) {
- ix_xmit_stop --;
- if(ix_xmit_stop == 0) fprintf(stderr,"Restarting Dolphin Xmit\n");
- }
-
- }
- sprintf(dcstatus,"%ld ",ets);
- for(ii=0;ii<mytotaldcu;ii++) {
- sprintf(dcs,"%d %d %d ",edcuid[ii],estatus[ii],edbs[ii]);
- strcat(dcstatus,dcs);
- }
-
-
- // Increment cycle count
- nextCycle ++;
- nextCycle %= 16;
- }while (keepRunning); // End of infinite loop
-
- // Stop Rcv Threads
- fprintf(stderr,"stopping threads %d \n",nsys);
- stop_working_threads = 1;
-
- // Wait for threads to stop
- sleep(5);
- fprintf(stderr,"closing out MX\n");
- mx_finalize();
- if(xmitData) {
- fprintf(stderr,"closing out ix\n");
- // Cleanup the Dolphin connections
- error = dolphin_closeout();
- }
-
- exit(0);
-}
diff --git a/src/ix_stream/mx2rcv.c b/src/ix_stream/mx2rcv.c
deleted file mode 100644
index 2e75f819f..000000000
--- a/src/ix_stream/mx2rcv.c
+++ /dev/null
@@ -1,823 +0,0 @@
-//
-/// @file mx2ix.c
-/// @brief DAQ data concentrator code. Receives data via Open-MX and sends via Dolphin IX..
-//
-
-#include "myriexpress.h"
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/file.h>
-#include <stdio.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include "../drv/crc.c"
-#include <time.h>
-#include "../include/daqmap.h"
-#include "../include/daq_core.h"
-#include "../zmq_stream/dc_utils.h"
-
-// #include "testlib.h"
-
-#include "../zmq_stream/simple_pv.h"
-
-#define __CDECL
-
-#define DO_HANDSHAKE 0
-
-
-
-#define MIN_DELAY_MS 5
-#define MAX_DELAY_MS 40
-
-#define MAX_FE_COMPUTERS 32
-
-#define MX_MUTEX_T pthread_mutex_t
-#define MX_MUTEX_INIT(mutex_) pthread_mutex_init(mutex_, 0)
-#define MX_MUTEX_LOCK(mutex_) pthread_mutex_lock(mutex_)
-#define MX_MUTEX_UNLOCK(mutex_) pthread_mutex_unlock(mutex_)
-
-#define MX_THREAD_T pthread_t
-#define MX_THREAD_CREATE(thread_, start_routine_, arg_) \
-pthread_create(thread_, 0, start_routine_, arg_)
-#define MX_THREAD_JOIN(thread_) pthread_join(thread, 0)
-
-MX_MUTEX_T stream_mutex;
-
-#define FILTER 0x12345
-#define MATCH_VAL 0xabcdef
-#define DFLT_EID 1
-#define DFLT_LEN 8192
-#define DFLT_END 128
-#define MAX_LEN (1024*1024*1024)
-#define DFLT_ITER 1000
-#define NUM_RREQ 16 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-#define NUM_SREQ 256 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-
-#define DO_HANDSHAKE 0
-#define MATCH_VAL_MAIN (1 << 31)
-#define MATCH_VAL_THREAD 1
-#define THREADS_PER_NIC 16
-#define IX_STOP_SEC 5
-
-// daq_multi_cycle_header_t *xmitHeader[IX_BLOCK_COUNT];
-
-extern void *findSharedMemorySize(char *,int);
-
-int do_verbose = 0;
-
-struct thread_info {
- int index;
- uint32_t match_val;
- uint32_t tpn;
- uint32_t bid;
-};
-struct thread_mon_info {
- int index;
- void *ctx;
-};
-struct thread_info thread_index[DCU_COUNT];
-char *sname[DCU_COUNT]; // Names of FE computers serving DAQ data
-char *local_iface[MAX_FE_COMPUTERS];
-daq_multi_dcu_data_t mxDataBlockSingle[MAX_FE_COMPUTERS];
-int64_t dataRecvTime[MAX_FE_COMPUTERS];
-const int mc_header_size = sizeof(daq_multi_cycle_header_t);
-int stop_working_threads = 0;
-int start_acq = 0;
-static volatile int keepRunning = 1;
-int thread_cycle[MAX_FE_COMPUTERS];
-int thread_timestamp[MAX_FE_COMPUTERS];
-int rcv_errors = 0;
-int dataRdy[MAX_FE_COMPUTERS];
-
-void
-usage()
-{
- fprintf(stderr, "Usage: mx2ix [args] -s server names -m shared memory size -g IX channel \n");
- fprintf(stderr, "-l filename - log file name\n");
- fprintf(stderr, "-s - number of FE computers to connect (1-32): Default = 32\n");
- fprintf(stderr, "-v - verbose prints diag test data\n");
- fprintf(stderr, "-g - Dolphin IX channel to xmit on (0-3)\n");
- fprintf(stderr, "-p - Debug pv prefix, requires -P as well\n");
- fprintf(stderr, "-P - Path to a named pipe to send PV debug information to\n");
- fprintf(stderr, "-d - Max delay in milli seconds to wait for a FE to send data, defaults to 10\n");
- fprintf(stderr, "-t - Number of rcvr threads per NIC: default = 16\n");
- fprintf(stderr, "-n - Data Concentrator number (0 or 1) : default = 0\n");
- fprintf(stderr, "-h - help\n");
-}
-
-// *************************************************************************
-// Timing Diagnostic Routine
-// *************************************************************************
-static int64_t
-s_clock (void)
-{
-struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-// *************************************************************************
-// Catch Control C to end cod in controlled manner
-// *************************************************************************
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-void sigpipeHandler(int dummy)
-{}
-
-// **********************************************************************************************
-void print_diags(int nsys, int lastCycle, int sendLength, daq_multi_dcu_data_t *ixDataBlock,int dbs[]) {
-// **********************************************************************************************
- int ii = 0;
- // Print diags in verbose mode
- fprintf(stderr,"Receive errors = %d\n",rcv_errors);
- fprintf(stderr,"Time = %d\t size = %d\n",ixDataBlock->header.dcuheader[0].timeSec,sendLength);
- fprintf(stderr,"DCU ID\tCycle \t TimeSec\tTimeNSec\tDataSize\tTPCount\tTPSize\tXmitSize\n");
- for(ii=0;ii<nsys;ii++) {
- fprintf(stderr,"%d",ixDataBlock->header.dcuheader[ii].dcuId);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].cycle);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].timeSec);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].timeNSec);
- fprintf(stderr,"\t\t%d",ixDataBlock->header.dcuheader[ii].dataBlockSize);
- fprintf(stderr,"\t\t%d",ixDataBlock->header.dcuheader[ii].tpCount);
- fprintf(stderr,"\t%d",ixDataBlock->header.dcuheader[ii].tpBlockSize);
- fprintf(stderr,"\t%d",dbs[ii]);
- fprintf(stderr,"\n ");
- }
-}
-
-
-// *************************************************************************
-// Thread for receiving DAQ data via Open-MX
-// *************************************************************************
-void *rcvr_thread(void *arg) {
- struct thread_info* my_info = (struct thread_info*)arg;
- int mt = my_info->index;
- uint32_t mv = my_info->match_val;
- uint32_t board_id = my_info->bid;
- uint32_t tpn = my_info->tpn;
- int cycle = 0;
- daq_multi_dcu_data_t *mxDataBlock;
- mx_return_t ret;
- int count, len, cur_req;
- mx_status_t stat;
- mx_request_t req[NUM_RREQ];
- mx_segment_t seg;
- uint32_t result;
- char *buffer;
- uint32_t filter;
- mx_endpoint_t ep;
- int myErrorStat = 0;
-
- filter = FILTER;
- int copySize;
-
-
- int dblock = board_id * tpn + mt;
- fprintf(stderr,"Starting receive loop for thread %d %d\n", board_id,mt);
-
- ret = mx_open_endpoint(board_id, mt, filter, NULL, 0, &ep);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to open endpoint %s\n", mx_strerror(ret));
- return(0);
- }
-
- fprintf(stderr,"waiting for someone to connect on CH %d\n",mt);
- len = 0xf00000;
- fprintf(stderr,"buffer length = %d\n",len);
- buffer = (char *)malloc(len);
- if (buffer == NULL) {
- fprintf(stderr, "Can't allocate buffers here\n");
- mx_close_endpoint(ep);
- return(0);
- }
- len /= NUM_RREQ;
- fprintf(stderr," length = %d\n",len);
-
- for (cur_req = 0; cur_req < NUM_RREQ; cur_req++) {
- seg.segment_ptr = &buffer[cur_req * len];
- seg.segment_length = len;
- mx_irecv(ep, &seg, 1, mv, MX_MATCH_MASK_NONE, 0,
- &req[cur_req]);
- }
-
- mx_set_error_handler(MX_ERRORS_RETURN);
-
- char *daqbuffer = (char *)&mxDataBlockSingle[dblock];
- do {
- for (count = 0; count < NUM_RREQ; count++) {
- cur_req = count & (16 - 1);
- mx_wait(ep, &req[cur_req],150, &stat, &result);
- myErrorStat = 0;
- if (!result) {
- myErrorStat = 1;
- count --;
- }
- if (stat.code != MX_STATUS_SUCCESS) {
- myErrorStat = 2;
- }
- if(!myErrorStat) {
- seg.segment_ptr = &buffer[cur_req * len];
- mxDataBlock = (daq_multi_dcu_data_t *)seg.segment_ptr;
- copySize = stat.xfer_length;
- memcpy(daqbuffer,seg.segment_ptr,copySize);
- // Get the message DAQ cycle number
- cycle = mxDataBlock->header.dcuheader[0].cycle;
- // Pass cycle and timestamp data back to main process
- thread_cycle[dblock] = cycle;
- thread_timestamp[dblock] = mxDataBlock->header.dcuheader[0].timeSec;
- dataRecvTime[dblock] = s_clock();
- mx_irecv(ep, &seg, 1, mv, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- }
-
- }
- // Run until told to stop by main thread
- } while(!stop_working_threads);
- fprintf(stderr,"Stopping thread %d\n",mt);
- usleep(200000);
-
- mx_close_endpoint(ep);
- return(0);
-
-}
-
-// *************************************************************************
-// Main Process
-// *************************************************************************
-int
-main(int argc, char **argv)
-{
- pthread_t thread_id[MAX_FE_COMPUTERS];
- unsigned int nsys = MAX_FE_COMPUTERS; // The number of mapped shared memories (number of data sources)
- char *buffer_name = "ifo";
- int c;
- int ii; // Loop counter
- int delay_ms = 10;
- int delay_cycles = 0;
-
- extern char *optarg; // Needed to get arguments to program
-
- // PV/debug information
- char *pv_prefix = 0;
- char *pv_debug_pipe_name = 0;
- int pv_debug_pipe = -1;
-
- // Declare shared memory data variables
- daq_multi_cycle_header_t *ifo_header;
- char *ifo;
- char *ifo_data;
- int cycle_data_size;
- daq_multi_dcu_data_t *ifoDataBlock;
- char *nextData;
- int max_data_size_mb = 100;
- int max_data_size = 0;
- char *mywriteaddr;
- int dc_number = 1;
-
-
- /* set up defaults */
- int xmitData = 0;
- int tpn = THREADS_PER_NIC;
-
-
-
- // Get arguments sent to process
- while ((c = getopt(argc, argv, "b:hs:m:g:vp:P:d:l:t:n:")) != EOF) switch(c) {
- case 's':
- nsys = atoi(optarg);
- if(nsys > MAX_FE_COMPUTERS) {
- fprintf(stderr,"Max number of FE computers is 32\n");
- usage();
- exit(1);
- }
- break;
- case 'v':
- do_verbose = 1;
- break;
- case 'n':
- dc_number = atoi(optarg);
- dc_number ++;
- if(dc_number > 2) {
- fprintf(stderr,"DC number must be 0 or 1\n");
- usage();
- exit(1);
- }
- break;
- case 'm':
- max_data_size_mb = atoi(optarg);
- if (max_data_size_mb < 20){
- fprintf(stderr,"Min data block size is 20 MB\n");
- return -1;
- }
- if (max_data_size_mb > 100){
- fprintf(stderr,"Max data block size is 100 MB\n");
- return -1;
- }
- break;
- case 't':
- tpn = atoi(optarg);
- break;
- case 'b':
- buffer_name = optarg;
- break;
- case 'p':
- pv_prefix = optarg;
- break;
- case 'P':
- pv_debug_pipe_name = optarg;
- break;
- case 'd':
- delay_ms = atoi(optarg);
- if (delay_ms < MIN_DELAY_MS || delay_ms > MAX_DELAY_MS) {
- fprintf(stderr,"The delay factor must be between 5ms and 40ms\n");
- return -1;
- }
- break;
- case 'l':
- if (0 == freopen(optarg, "w", stdout)) {
- perror ("freopen");
- exit (1);
- }
- setvbuf(stdout, NULL, _IOLBF, 0);
- stderr = stdout;
- break;
- case 'h':
- default:
- usage();
- exit(1);
- }
- max_data_size = max_data_size_mb * 1024*1024;
- delay_cycles = delay_ms * 10;
-
- mx_init();
- // MX_MUTEX_INIT(&stream_mutex);
-
- // set up to catch Control C
- signal(SIGINT,intHandler);
- // setup to ignore sig pipe
- signal(SIGPIPE, sigpipeHandler);
-
- fprintf(stderr,"Num of sys = %d\n",nsys);
-
- // Get pointers to local DAQ mbuf
- ifo = (char *)findSharedMemorySize(buffer_name,max_data_size_mb);
- ifo_header = (daq_multi_cycle_header_t *)ifo;
- ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
- cycle_data_size = (max_data_size - sizeof(daq_multi_cycle_header_t)) / DAQ_NUM_DATA_BLOCKS_PER_SECOND;
- cycle_data_size -= (cycle_data_size % 8);
- fprintf (stderr,"cycle data size = %d\t%d\n",cycle_data_size, max_data_size_mb);
- sleep(3);
- ifo_header->cycleDataSize = cycle_data_size;
- ifo_header->maxCycle = DAQ_NUM_DATA_BLOCKS_PER_SECOND;
-
-#if 0
- if(xmitData) {
- // Connect to Dolphin
- error = dolphin_init();
- fprintf(stderr,"Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- // Set pointer to xmit header in Dolphin xmit data area.
- mywriteaddr = (char *)writeAddr;
- for(ii=0;ii<IX_BLOCK_COUNT;ii++) {
- xmitHeader[ii] = (daq_multi_cycle_header_t *)mywriteaddr;
- mywriteaddr += IX_BLOCK_SIZE;
- xmitDataOffset[ii] = IX_BLOCK_SIZE * ii + sizeof(struct daq_multi_cycle_header_t);
- fprintf(stderr,"Dolphin at 0x%lx and 0x%lx",(long)xmitHeader[ii],(long)xmitDataOffset[ii]);
- }
- }
-#endif
-
- fprintf(stderr,"nsys = %d\n",nsys);
-
- // Make 0MQ socket connections
- for(ii=0;ii<nsys;ii++) {
- // Create a thread to receive data from each data server
- thread_index[ii].index = ii % tpn;
- thread_index[ii].match_val = MATCH_VAL_MAIN;
- thread_index[ii].tpn = tpn;
- thread_index[ii].bid = ii/tpn;
- pthread_create(&thread_id[ii],NULL,rcvr_thread,(void *)&thread_index[ii]);
-
- }
- int nextCycle = 0;
- start_acq = 1;
- int64_t mytime = 0;
- int64_t mylasttime = 0;
- int64_t myptime = 0;
- int64_t n_cycle_time = 0;
- int mytotaldcu = 0;
- char *zbuffer;
- size_t zbuffer_remaining = 0;
- int dc_datablock_size = 0;
- int datablock_size_running = 0;
- int datablock_size_mb_s = 0;
- static const int header_size = sizeof(daq_multi_dcu_header_t);
- char dcstatus[4096];
- char dcs[48];
- int edcuid[10];
- int estatus[10];
- int edbs[10];
- unsigned long ets = 0;
- int timeout = 0;
- int threads_rdy;
- int any_rdy = 0;
- int jj,kk;
- int sendLength = 0;
-
- int min_cycle_time = 1 << 30;
- int pv_min_cycle_time = 0;
- int max_cycle_time = 0;
- int pv_max_cycle_time = 0;
- int mean_cycle_time = 0;
- int pv_mean_cycle_time = 0;
- int pv_dcu_count = 0;
- int pv_total_datablock_size = 0;
- int pv_datablock_size_mb_s = 0;
- int uptime = 0;
- int pv_uptime = 0;
- int gps_time = 0;
- int pv_gps_time = 0;
- int missed_flag = 0;
- int missed_nsys[MAX_FE_COMPUTERS];
- int64_t recv_time[MAX_FE_COMPUTERS];
- int64_t min_recv_time = 0;
- int recv_buckets[(MAX_DELAY_MS/5)+2];
- int festatus = 0;
- int pv_festatus = 0;
- int ix_xmit_stop = 0;
- SimplePV pvs[] = {
- {
- "RECV_MIN_MS",
- SIMPLE_PV_INT,
- &pv_min_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MAX_MS",
- SIMPLE_PV_INT,
- &pv_max_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MEAN_MS",
- SIMPLE_PV_INT,
- &pv_mean_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "DCU_COUNT",
- SIMPLE_PV_INT,
- &pv_dcu_count,
-
- 120,
- 0,
- 115,
- 0,
- },
- {
- "DATA_SIZE",
- SIMPLE_PV_INT,
- &pv_total_datablock_size,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1*1024*1024,
- },
- {
- "DATA_RATE",
- SIMPLE_PV_INT,
- &pv_datablock_size_mb_s,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1000000,
- },
- {
- "UPTIME_SECONDS",
- SIMPLE_PV_INT,
- &pv_uptime,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
-
- 1,
- },
- {
- "RCV_STATUS",
- SIMPLE_PV_INT,
- &pv_festatus,
-
- 0xffffffff,
- 0,
- 0xfffffffe,
-
- 1,
- },
- {
- "GPS",
- SIMPLE_PV_INT,
- &pv_gps_time,
-
- 0xfffffff,
- 0,
- 0xfffffffe,
-
- 1,
- },
-
- };
- if (pv_debug_pipe_name)
- {
- pv_debug_pipe = open(pv_debug_pipe_name, O_NONBLOCK | O_RDWR, 0);
- if (pv_debug_pipe < 0) {
- fprintf(stderr, "Unable to open %s for writting (pv status)\n", pv_debug_pipe_name);
- exit(1);
- }
- }
-
- missed_flag = 1;
- memset(&missed_nsys[0], 0, sizeof(missed_nsys));
- memset(recv_buckets, 0, sizeof(recv_buckets));
- do {
- // Reset counters
- timeout = 0;
- for(ii=0;ii<nsys;ii++) dataRdy[ii] = 0;
- for(ii=0;ii<nsys;ii++) thread_cycle[ii] = 50;
- threads_rdy = 0;
- any_rdy = 0;
-
- // Wait up to 100ms until received data from at least 1 FE or timeout
- do {
- usleep(2000);
- for(ii=0;ii<nsys;ii++) {
- if(nextCycle == thread_cycle[ii]) any_rdy = 1;
- }
- timeout += 1;
- }while(!any_rdy && timeout < 50);
-#ifndef TIME_INTERVAL_DIAG
- mytime = s_clock();
-#endif
-
- // Wait up to delay_ms ms in 1/10ms intervals until data received from everyone or timeout
- timeout = 0;
- do {
- usleep(100);
- for(ii=0;ii<nsys;ii++) {
- if(nextCycle == thread_cycle[ii] && !dataRdy[ii]) threads_rdy ++;
- if(nextCycle == thread_cycle[ii]) dataRdy[ii] = 1;
- }
- timeout += 1;
- }while(threads_rdy < nsys && timeout < delay_cycles);
- if(timeout >= 100) rcv_errors += (nsys - threads_rdy);
-#ifndef TIME_INTERVAL_DIAG
- mylasttime = s_clock();
-#endif
-
- if(any_rdy) {
- int tbsize = 0;
-#ifdef TIME_INTERVAL_DIAG
- // Timing diagnostics for time between cycles
- mytime = s_clock();
- myptime = mytime - mylasttime;
- mylasttime = mytime;
-#else
- // Timing diagnostics for rcv window
- myptime = mylasttime - mytime;
-#endif
-
- if (myptime < min_cycle_time) {
- min_cycle_time = myptime;
- }
- if (myptime > max_cycle_time) {
- max_cycle_time = myptime;
- }
- mean_cycle_time += myptime;
- ++n_cycle_time;
-
- // Reset total DCU counter
- mytotaldcu = 0;
- // Reset total DC data size counter
- dc_datablock_size = 0;
- // Get pointer to next data block in shared memory
- nextData = (char *)ifo_data;
- nextData += cycle_data_size * nextCycle;
- ifoDataBlock = (daq_multi_dcu_data_t *)nextData;
- zbuffer = (char *)nextData + header_size;
- zbuffer_remaining = cycle_data_size - header_size;
-
- min_recv_time = 0x7fffffffffffffff;
- festatus = 0;
- // Loop over all data buffers received from FE computers
- for(ii=0;ii<nsys;ii++) {
- recv_time[ii] = -1;
- if(dataRdy[ii]) {
- festatus += (1 << ii);
- recv_time[ii] = dataRecvTime[ii];
- if (recv_time[ii] < min_recv_time) {
- min_recv_time = recv_time[ii];
- }
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "+++%d\n", ii);
- }
- int myc = mxDataBlockSingle[ii].header.dcuTotalModels;
- // For each model, copy over data header information
- for(jj=0;jj<myc;jj++) {
- // Copy data header information
- ifoDataBlock->header.dcuheader[mytotaldcu].dcuId =
- mxDataBlockSingle[ii].header.dcuheader[jj].dcuId;
- ifoDataBlock->header.dcuheader[mytotaldcu].fileCrc =
- mxDataBlockSingle[ii].header.dcuheader[jj].fileCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].status =
- mxDataBlockSingle[ii].header.dcuheader[jj].status;
- ifoDataBlock->header.dcuheader[mytotaldcu].cycle =
- mxDataBlockSingle[ii].header.dcuheader[jj].cycle;
- if(!ix_xmit_stop) {
- ifoDataBlock->header.dcuheader[mytotaldcu].timeSec =
- mxDataBlockSingle[ii].header.dcuheader[jj].timeSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeNSec =
- mxDataBlockSingle[ii].header.dcuheader[jj].timeNSec;
- }
- ifoDataBlock->header.dcuheader[mytotaldcu].dataCrc =
- mxDataBlockSingle[ii].header.dcuheader[jj].dataCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataBlockSize =
- mxDataBlockSingle[ii].header.dcuheader[jj].dataBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpBlockSize =
- mxDataBlockSingle[ii].header.dcuheader[jj].tpBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount =
- mxDataBlockSingle[ii].header.dcuheader[jj].tpCount;
-
- // Check for downstream DAQ reset request from EDCU
- if(mxDataBlockSingle[ii].header.dcuheader[jj].dcuId == 52 &&
- mxDataBlockSingle[ii].header.dcuheader[jj].tpCount == dc_number)
- {
- fprintf(stderr,"Got a DAQ Reset REQUEST %u\n",mxDataBlockSingle[ii].header.dcuheader[jj].timeSec);
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount = 0;
- ix_xmit_stop = 16 * IX_STOP_SEC;
- }
- for(kk=0;kk<DAQ_GDS_MAX_TP_NUM ;kk++)
- ifoDataBlock->header.dcuheader[mytotaldcu].tpNum[kk] = mxDataBlockSingle[ii].header.dcuheader[jj].tpNum[kk];
- edbs[mytotaldcu] = mxDataBlockSingle[ii].header.dcuheader[jj].tpBlockSize + mxDataBlockSingle[ii].header.dcuheader[jj].dataBlockSize;
- // Get some diags
- if(ifoDataBlock->header.dcuheader[mytotaldcu].status != 0xbad)
- ets = mxDataBlockSingle[ii].header.dcuheader[jj].timeSec;
- estatus[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].status;
- edcuid[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].dcuId;
- // Increment total DCU count
- mytotaldcu ++;
- }
- // Get the size of the data to transfer
- int mydbs = mxDataBlockSingle[ii].header.fullDataBlockSize;
- // Get pointer to data in receive data block
- char *mbuffer = (char *)&mxDataBlockSingle[ii].dataBlock[0];
- if (mydbs > zbuffer_remaining)
- {
- fprintf(stderr,"Buffer overflow found. Attempting to write %d bytes to zbuffer which has %d bytes remainging\n",
- (int)mydbs, (int)zbuffer_remaining);
- abort();
- }
- // Copy data from receive buffer to shared memory
- memcpy(zbuffer,mbuffer,mydbs);
- // Increment shared memory data buffer pointer for next data set
- zbuffer += mydbs;
- // Calc total size of data block for this cycle
- dc_datablock_size += mydbs;
- tbsize += mydbs;
- } else {
- missed_nsys[ii] |= missed_flag;
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "---%d\n", ii);
- }
- }
- }
- // Write total data block size to shared memory header
- ifoDataBlock->header.fullDataBlockSize = dc_datablock_size;
- // Write total dcu count to shared memory header
- ifoDataBlock->header.dcuTotalModels = mytotaldcu;
- // Set multi_cycle head cycle to indicate data ready for this cycle
- ifo_header->curCycle = nextCycle;
-
- // Calc IX message size
- sendLength = header_size + ifoDataBlock->header.fullDataBlockSize;
- for (ii = 0; ii < nsys; ++ii) {
- recv_time[ii] -= min_recv_time;
- }
- datablock_size_running += dc_datablock_size;
- if (nextCycle == 0) {
- datablock_size_mb_s = datablock_size_running / 1024 ;
- pv_datablock_size_mb_s = datablock_size_mb_s;
- uptime ++;
- pv_uptime = uptime;
- gps_time = ifoDataBlock->header.dcuheader[0].timeSec;
- pv_gps_time = gps_time;
- pv_dcu_count = mytotaldcu;
- pv_festatus = festatus;
- mean_cycle_time = (n_cycle_time > 0 ? mean_cycle_time / n_cycle_time : 1 << 31);
-
- pv_mean_cycle_time = mean_cycle_time;
- pv_max_cycle_time = max_cycle_time;
- pv_min_cycle_time = min_cycle_time;
- send_pv_update(pv_debug_pipe, pv_prefix, pvs, sizeof(pvs)/sizeof(pvs[0]));
-
- if (do_verbose) {
- fprintf(stderr,"\nData rdy for cycle = %d\t\tTime Interval = %ld msec\n", nextCycle, myptime);
- fprintf(stderr,"Min/Max/Mean cylce time %d/%d/%d msec over %ld cycles\n", min_cycle_time, max_cycle_time,
- mean_cycle_time, n_cycle_time);
- fprintf(stderr,"Total DCU = %d\t\t\tBlockSize = %d\n", mytotaldcu, dc_datablock_size);
- print_diags(mytotaldcu, nextCycle, sendLength, ifoDataBlock, edbs);
- }
- n_cycle_time = 0;
- min_cycle_time = 1 << 30;
- max_cycle_time = 0;
- mean_cycle_time = 0;
-
-
- missed_flag = 1;
- datablock_size_running = 0;
- } else {
- missed_flag <<= 1;
- }
-#if 0
- if(xmitData && !ix_xmit_stop) {
- if (sendLength > IX_BLOCK_SIZE)
- {
- fprintf(stderr, "Buffer overflow. Sending %d bytes into a dolphin block that holds %d\n",
- (int)sendLength, (int)IX_BLOCK_SIZE);
- abort();
- }
- // WRITEDATA to Dolphin Network
- SCIMemCpy(sequence,nextData, remoteMap,xmitDataOffset[xmitBlockNum],sendLength,memcpyFlag,&error);
- error = SCI_ERR_OK;
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- fprintf(stderr,"For reference the expected error codes are:\n");
- fprintf(stderr,"SCI_ERR_OUT_OF_RANGE = 0x%x\n", SCI_ERR_OUT_OF_RANGE);
- fprintf(stderr,"SCI_ERR_SIZE_ALIGNMENT = 0x%x\n", SCI_ERR_SIZE_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_OFFSET_ALIGNMENT = 0x%x\n", SCI_ERR_OFFSET_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_TRANSFER_FAILED = 0x%x\n", SCI_ERR_TRANSFER_FAILED);
- return error;
- }
- // Set data header information
- unsigned int maxCycle = ifo_header->maxCycle;
- unsigned int curCycle = ifo_header->curCycle;
- xmitHeader[xmitBlockNum]->maxCycle = maxCycle;
- xmitHeader[xmitBlockNum]->cycleDataSize = sendLength;;
- // Send cycle last as indication of data ready for receivers
- xmitHeader[xmitBlockNum]->curCycle = curCycle;
- // Have to flush the buffers to make data go onto Dolphin network
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-#endif
- if(ix_xmit_stop) {
- ix_xmit_stop --;
- if(ix_xmit_stop == 0) fprintf(stderr,"Restarting Dolphin Xmit\n");
- }
-
- }
- sprintf(dcstatus,"%ld ",ets);
- for(ii=0;ii<mytotaldcu;ii++) {
- sprintf(dcs,"%d %d %d ",edcuid[ii],estatus[ii],edbs[ii]);
- strcat(dcstatus,dcs);
- }
-
-
- // Increment cycle count
- nextCycle ++;
- nextCycle %= 16;
- }while (keepRunning); // End of infinite loop
-
- // Stop Rcv Threads
- fprintf(stderr,"stopping threads %d \n",nsys);
- stop_working_threads = 1;
-
- // Wait for threads to stop
- sleep(5);
- fprintf(stderr,"closing out MX\n");
- mx_finalize();
-
- exit(0);
-}
diff --git a/src/ix_stream/mx_rcv.cc b/src/ix_stream/mx_rcv.cc
deleted file mode 100644
index 4bde33688..000000000
--- a/src/ix_stream/mx_rcv.cc
+++ /dev/null
@@ -1,901 +0,0 @@
-// Myrinet MX DQ data receiver
-
-#include <pthread.h>
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <iostream>
-#include <fstream>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "dolphin_common.hh"
-
-#include "mx_extensions.h"
-#include "myriexpress.h"
-
-#include "../include/daqmap.h"
-#include "daq_core.h"
-#include "recvr_utils.hh"
-
-const int MXR_MAX_DCU = 128;
-const int MAX_RECEIVE_BUFFERS = 16; // number of 16Hz cycles we are buffering
-
-const char* DEFAULT_BUFFER_NAME = "ifo";
-const unsigned int DEFAULT_MAX_DATA_SIZE_MB = 100;
-
-extern "C" {
-extern volatile void *findSharedMemorySize(const char *, int);
-}
-
-enum DEBUG_FLAG {
- DEBUG_FALSE = 0,
- DEBUG_TRUE,
-};
-
-typedef flag_mask<MXR_MAX_DCU> dcu_mask_t;
-
-/**
- * Program options
- */
-struct options_t {
- std::string buffer_name;
- std::string receive_map_dump_fname;
- unsigned int max_data_size_mb;
- unsigned int dolphin_group;
- bool abort;
- bool xmit_data;
-
- options_t():
- buffer_name(DEFAULT_BUFFER_NAME),
- receive_map_dump_fname(),
- max_data_size_mb(DEFAULT_MAX_DATA_SIZE_MB),
- dolphin_group(-1),
- abort(false),
- xmit_data(false)
- {}
-};
-
-struct local_mx_info {
- uint32_t nics_available;
- uint32_t max_endpoints;
- uint32_t max_requests;
-};
-
-/**
- * Diagnostic, return the system time in ms
- */
-static int64_t
-s_clock (void)
-{
- struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-/**
- * daqMXdata, the format the data arrives in
- */
-struct daqMXdata {
- struct rmIpcStr mxIpcData;
- cdsDaqNetGdsTpNum mxTpTable;
- char mxDataBlock[DAQ_DCU_BLOCK_SIZE];
-};
-
-/**
- * single_dcu_block the format we keep the data in.
- * We also maintain a lock that should be used when accessing
- * this data.
- */
-struct single_dcu_block {
- daq_msg_header_t header;
- char data[DAQ_DCU_BLOCK_SIZE];
- spin_lock lock_;
-
- single_dcu_block(): header(), data(), lock_()
- {
- }
-
- void lock()
- {
- lock_.lock();
- }
- void unlock()
- {
- lock_.unlock();
- }
-private:
- single_dcu_block(const single_dcu_block& other);
- single_dcu_block& operator=(const single_dcu_block& other);
-};
-
-/**
- * Simple structure used to get a mapping of which dcus did not report
- * in the current cycle that did in the previous
- */
-struct missing_map_t {
- missing_map_t(): missing_map(), differences(0)
- {
- std::fill(&missing_map[0], &missing_map[0] + MXR_MAX_DCU, false);
- }
- void operator()(const bool old_entry, const bool new_entry, int i)
- {
- missing_map[i] = old_entry;
- differences += (old_entry ? 1 : 0);
- }
- bool missing_map[MXR_MAX_DCU];
- int differences;
-};
-
-typedef strong_type<int, 0> buffer_index_t;
-typedef strong_type<int, 1> dcu_index_t;
-typedef debug_array_2d<receive_map_entry_t, MAX_RECEIVE_BUFFERS, MXR_MAX_DCU, buffer_index_t, dcu_index_t> receive_map_t;
-
-debug_array_2d<single_dcu_block, MXR_MAX_DCU, MAX_RECEIVE_BUFFERS, dcu_index_t, buffer_index_t> mxr_data;
-receive_map_t receive_map;
-spin_lock receive_map_locks[MAX_RECEIVE_BUFFERS];
-
-atomic_flag threads_should_stop;
-atomic_flag exit_main_loop;
-
-buffer_index_t cycle_to_buffer_index(int cycle)
-{
- return buffer_index_t(cycle % MAX_RECEIVE_BUFFERS);
-}
-
-void mark_dcu_received(int dcuid, int64_t gps_sec, int cycle)
-{
- if (dcuid < 0 || dcuid >= MXR_MAX_DCU)
- {
- return;
- }
- receive_map_entry_t new_entry;
- new_entry.s_clock = s_clock();
- new_entry.gps_sec_and_cycle = calc_gps_sec_and_cycle(gps_sec, cycle);
- buffer_index_t index = cycle_to_buffer_index(cycle);
- {
- //lock_guard<spin_lock> l_(receive_map_locks[get_value(index)]);
- lock_guard<spin_lock> l_(receive_map_locks[0]);
- receive_map[index][dcu_index_t(dcuid)] = new_entry;
- }
-}
-
-void receive_map_copy_row(int cycle, receive_map_entry_t dest[MXR_MAX_DCU])
-{
- buffer_index_t index = cycle_to_buffer_index(cycle);
- {
- //lock_guard<spin_lock> l_(receive_map_locks[get_value(index)]);
- lock_guard<spin_lock> l_(receive_map_locks[0]);
- std::copy(&(receive_map[index][0]), &(receive_map[index][MXR_MAX_DCU-1]), dest);
- }
-}
-
-void receive_map_copy(receive_map_t& dest)
-{
- lock_guard<spin_lock> l_(receive_map_locks[0]);
- dest = receive_map;
-}
-
-class scoreboard_manager {
-public:
- scoreboard_manager(receive_map_t& scoreboard_snapshot, dcu_mask_t& mask):scoreboard_snapshot_(scoreboard_snapshot), mask_(mask) {}
- void get_scoreboard_row(int cycle, receive_map_entry_t dest[MXR_MAX_DCU])
- {
- buffer_index_t cycle_index = cycle_to_buffer_index(cycle);
- receive_map_copy(scoreboard_snapshot_);
- std::copy(&(scoreboard_snapshot_[cycle_index][0]), &(scoreboard_snapshot_[cycle_index][MXR_MAX_DCU - 1]), dest);
- }
- void mark_received(int dcu)
- {
- mask_.set(dcu);
- }
-private:
- scoreboard_manager(const scoreboard_manager& other);
- scoreboard_manager& operator=(const scoreboard_manager& other);
-
- receive_map_t& scoreboard_snapshot_;
- dcu_mask_t& mask_;
-};
-
-void dump_scoreboard(std::ostream& os, receive_map_t& local_map, missing_map_t& missing_map, int64_t sec_and_cycle)
-{
- // figure out which columns have entries
- int64_t column_has_data[MXR_MAX_DCU];
- std::fill(&column_has_data[0], &column_has_data[0] + MXR_MAX_DCU, 0);
- {
- for (int i = 0; i < MAX_RECEIVE_BUFFERS; ++i)
- {
- buffer_index_t index = cycle_to_buffer_index(i);
- for (int j = 0; j < MXR_MAX_DCU; ++j)
- {
- column_has_data[j] |= local_map[index][dcu_index_t(j)].gps_sec_and_cycle;
- }
- }
- }
- os << "-----------------------------\n";
- os << "- " << sec_and_cycle << " (" << extract_gps(sec_and_cycle);
- os << ":" << extract_cycle(sec_and_cycle) <<")\n";
- os << "-----------------------------\n";
-
- buffer_index_t cur_cycle = cycle_to_buffer_index(static_cast<int>(extract_cycle(sec_and_cycle)));
-
- for (int i = 0; i < MAX_RECEIVE_BUFFERS; ++i)
- {
- buffer_index_t cycle_index = cycle_to_buffer_index(i);
- os << i << ") ";
- for (int j = 0; j < MXR_MAX_DCU; ++j)
- {
- if (column_has_data[j] == 0)
- {
- continue;
- }
- receive_map_entry_t& cur_entry = local_map[cycle_index][dcu_index_t(j)];
- os << "| " << j << "] ";
- os << (cur_cycle.get() == cycle_index.get() && missing_map.missing_map[j] ? "*" : " ");
- os << extract_gps(cur_entry.gps_sec_and_cycle) << ":" << extract_cycle(cur_entry.gps_sec_and_cycle);
- os << " (" << cur_entry.s_clock << ") ";
- }
- os << "|\n";
- }
- os << "-----------------------------\n";
- os << std::endl;
-}
-
-/// Define max boards, endpoints for mx_rcvr
-#define MX_MAX_BOARDS 4
-#define MX_MAX_ENDPOINTS 32
-
-#define FILTER 0x12345
-#define MATCH_VAL 0xabcdef
-#define DFLT_EID 1
-#define DFLT_LEN 8192
-#define DFLT_END 128
-#define MAX_LEN (1024 * 1024 * 1024)
-#define DFLT_ITER 1000
-#define NUM_RREQ 256
-#define NUM_SREQ 4 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-
-#define MATCH_VAL_MAIN (1 << 31)
-#define MATCH_VAL_THREAD 1
-
-/* A simple class used to ensure that when an endpoint is opened it is also closed. */
-class managed_endpoint {
-public:
- managed_endpoint(uint32_t board_num, uint32_t endpoint_id, uint32_t filter): endpoint_()
- {
- mx_return_t ret = mx_open_endpoint(board_num, endpoint_id, filter, NULL, 0, &endpoint_);
- if (ret != MX_SUCCESS)
- {
- std::cerr << "Failed to open board " << board_num << " endpoint " << endpoint_id << " error " << mx_strerror(ret) << "\n";
- exit(1);
- }
- }
- ~managed_endpoint()
- {
- mx_close_endpoint(endpoint_);
- }
- mx_endpoint_t get()
- {
- return endpoint_;
- }
-private:
- managed_endpoint(const managed_endpoint& other);
- managed_endpoint& operator=(const managed_endpoint& other);
-
- mx_endpoint_t endpoint_;
-};
-
-static const int buf_size = DAQ_DCU_BLOCK_SIZE * 2;
-
-/**
- * Handle a bad message from mx_wait.
- * @param neid
- * @param endpoint
- * @param stat
- */
-void handle_bad_message(int neid, mx_endpoint_t endpoint, const mx_status_t& stat)
-{
- mx_return_t ret = MX_SUCCESS;
-
- if (stat.code == MX_STATUS_SUCCESS)
- {
- return;
- }
- // Request completed, but bad code
- fprintf(stderr, "mx_wait failed in rcvr eid=%03x; "
- "wait did not complete; status code is "
- "%s\n",
- neid, mx_strstatus(stat.code));
- // mx_return_t ret = mx_cancel(ep[neid], &req[cur_req],
- // &result);
- mx_endpoint_addr_t ep_addr;
- ret = mx_get_endpoint_addr(endpoint, &ep_addr);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "mx_get_endpoint_addr() eid=%03x "
- "failed with status %s\n",
- neid, mx_strerror(ret));
- } else {
- ret = mx_disconnect(endpoint, ep_addr);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "mx_disconnect() eid=%03x failed "
- "with status %s\n",
- neid, mx_strerror(ret));
- } else {
- fprintf(stderr, "disconnected from the sender on "
- "endpoint %03x\n",
- neid);
- }
- }
-}
-
-unsigned int get_DQ_block_size(const struct rmIpcStr& ipc)
-{
- /* we overload the channelCount to be DQ block size,
- * it is not intuitive, so hide it in a function with
- * a better name.
- */
- return ipc.channelCount;
-}
-
-/**
- * Copy and convert the daqMXdata structure from a message into
- * the global dcu data store.
- * @param cur_data
- */
-void handle_received_message(const daqMXdata& cur_data, enum DEBUG_FLAG do_debug)
-{
- // printf("received one\n");
- int dcu_id = cur_data.mxIpcData.dcuId;
-
- // Skip bad inputs
- if (dcu_id < 0 || dcu_id > (MXR_MAX_DCU - 1))
- {
- return;
- }
- // daqd should skip dcus it doesn't know
- // about, we cannot
- // if (daqd.dcuSize[0][dcu_id] == 0) {
- // mx_irecv(ep[neid], &seg, 1, match_val,
- // MX_MATCH_MASK_NONE,
- // 0, &req[cur_req]);
- // continue; // Unconfigured DCU
- // }
-
- int64_t gps_sec = 0;
- int cycle = static_cast<int>(cur_data.mxIpcData.cycle);
- buffer_index_t cycle_index = cycle_to_buffer_index(cycle);
- unsigned int len = cur_data.mxIpcData.dataBlockSize;
-
- if (do_debug == DEBUG_TRUE && dcu_id == 7)
- {
- std::cout << "> " << cur_data.mxIpcData.bp[cycle].timeSec << ":" << cycle << "(" << cycle_index.get() << ") => ";
- }
-
- single_dcu_block& cur_dest = mxr_data[dcu_index_t(dcu_id)][cycle_index];
- {
- lock_guard<single_dcu_block> l_(cur_dest);
-
- char *dataSource = (char *)cur_data.mxDataBlock;
- char *dataDest = cur_dest.data;
-
- // Move the block data into the buffer
- memcpy(dataDest, dataSource, len);
-
- cur_dest.header.dcuId = static_cast<unsigned int>(dcu_id);
- cur_dest.header.fileCrc = cur_data.mxIpcData.crc;
- cur_dest.header.status = 2;
- cur_dest.header.cycle = static_cast<unsigned int>(cycle);
- gps_sec = cur_dest.header.timeSec = cur_data.mxIpcData.bp[cycle].timeSec;
- cur_dest.header.timeNSec = cur_data.mxIpcData.bp[cycle].timeNSec;
- cur_dest.header.dataCrc = cur_data.mxIpcData.bp[cycle].crc;
- cur_dest.header.dataBlockSize = get_DQ_block_size(cur_data.mxIpcData);
- cur_dest.header.tpBlockSize = len - get_DQ_block_size(cur_data.mxIpcData);
- cur_dest.header.tpCount = static_cast<unsigned int>(cur_data.mxTpTable.count);
- std::copy(cur_data.mxTpTable.tpNum, cur_data.mxTpTable.tpNum + cur_dest.header.tpCount, cur_dest.header.tpNum);
- }
- // do this in a scope where the lock on the data segment is not held, so we are never holding two locks
- mark_dcu_received(dcu_id, gps_sec, cycle);
- if (do_debug == DEBUG_TRUE && dcu_id == 7)
- {
- std::cout << gps_sec << ":" << cycle << " - crc - " << cur_data.mxIpcData.bp[cycle].crc << "\n";
- }
-}
-
-void receiver_mx(int neid, enum DEBUG_FLAG do_debug) {
- mx_status_t stat;
- mx_segment_t seg;
- uint32_t result;
- std::vector<daqMXdata> buffer(NUM_RREQ); // mx receive buffers
- int myErrorStat = 0;
- int copySize;
- // float *testData;
- mx_return_t ret = MX_SUCCESS;
- uint32_t match_val = MATCH_VAL_MAIN;
- mx_request_t req[NUM_RREQ];
-
- uint32_t board_num = (neid >> 8);
- uint32_t ep_num = neid & 0xff;
-
- int messages_received = 0;
-
- const int daqMXdata_len = sizeof(daqMXdata);
-
- std::cout << "about to open board " << board_num << " ep " << ep_num << " neid " << neid << "\n";
- managed_endpoint endpoint(board_num, ep_num, FILTER);
-
- /* pre-post our receives */
- for (int cur_req = 0; cur_req < NUM_RREQ; ++cur_req) {
- seg.segment_ptr = &buffer[cur_req];
- seg.segment_length = sizeof(daqMXdata);
-
- ret = mx_irecv(endpoint.get(), &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- if (ret != MX_SUCCESS)
- {
- std::cerr << "mx_irecv return an error: " << mx_strerror(ret) << "\n";
- }
- if (board_num != 0) {
- //std::cout << ".(" << neid << " " << cur_req << ")" << std::flush;
- }
- }
-
- mx_set_error_handler(MX_ERRORS_RETURN);
-
- std::cout << neid << " registered buffers and beginning wait loop\n";
-
- while (!threads_should_stop.is_set())
- {
- for(int cur_req = 0; cur_req < NUM_RREQ && !threads_should_stop.is_set(); ++cur_req) {
- result = 0;
- do
- {
- ret = mx_wait(endpoint.get(), &req[cur_req], 10, &stat, &result);
- if (ret != MX_SUCCESS) {
- std::cerr << "mx_wait on " << neid << " failed: " << mx_strerror(ret) << "\n";
- exit(1);
- }
- } while (!result);
- daqMXdata& cur_data = buffer[cur_req];
- seg.segment_ptr = &cur_data;
- seg.segment_length = daqMXdata_len;
-
- ++messages_received;
- //std::cout << neid << " received a message, #" << messages_received << "\n" << std::endl;
- //std::cout << "." << std::flush;
- if (stat.code == MX_STATUS_SUCCESS)
- {
- handle_received_message(cur_data, do_debug);
- }
- else
- {
- handle_bad_message(neid, endpoint.get(), stat);
- }
- memset(&req[cur_req], 0, sizeof(req[cur_req]));
- mx_irecv(endpoint.get(), &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- }
- // daqd.producer1.rcvr_stats[neid].tick();
-
- }
- std::cout << "mx_receiver " << neid << " completed\n";
- // exit(1);
-}
-
-void *receiver_thread(void *args)
-{
- std::cout << "calling receiver_mx with " << *reinterpret_cast<int*>(args) << "\n";
- receiver_mx(*reinterpret_cast<int*>(args), DEBUG_FALSE);
- return NULL;
-}
-
-
-/// Initialize MX library.
-/// Returns the maximum number of end-points supoprted in the system.
-local_mx_info open_mx() {
- static int mx_ep_opened = 0;
- static local_mx_info info;
- uint16_t my_eid;
- uint32_t board_id;
- uint32_t filter;
- char *sysname;
- int c;
- extern char *optarg;
- mx_return_t ret;
- int fd;
-
- if (mx_ep_opened)
- return info;
-
- printf("%ld\n", sizeof(struct daqMXdata));
-
- // So that openmx is not aborting on connection loss
- char omx_env_setting[] = "OMX_ERRORS_ARE_FATAL=1";
- char mx_env_setting[] = "MX_ERRORS_ARE_FATAL=1";
- putenv(omx_env_setting);
- putenv(mx_env_setting);
-
- ret = mx_init();
- if (ret != MX_SUCCESS)
- {
- fprintf(stderr, "mx_init failed: %s\n", mx_strerror(ret));
- exit(1);
- }
-
- /* set up defaults */
- sysname = NULL;
- filter = FILTER;
- ret = mx_get_info(0, MX_MAX_NATIVE_ENDPOINTS, 0, 0, &info.max_endpoints,
- sizeof(info.max_endpoints));
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to do mx_get_info: %s\n", mx_strerror(ret));
- exit(1);
- }
- fprintf(stderr, "MX has %d maximum end-points configured\n", info.max_endpoints);
- ret = mx_get_info(0, MX_NIC_COUNT, 0, 0, &info.nics_available,
- sizeof(info.nics_available));
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to do mx_get_info: %s\n", mx_strerror(ret));
- exit(1);
- }
- fprintf(stderr, "%d MX NICs available\n", info.nics_available);
- /// make sure these don't exceed array allocations
- if (info.max_endpoints > MX_MAX_ENDPOINTS) {
- fprintf(stderr,
- "ERROR: max end-points of %d exceeds array limit of %d\n",
- info.max_endpoints, MX_MAX_ENDPOINTS);
- exit(1);
- }
- if (info.nics_available > MX_MAX_BOARDS) {
- fprintf(stderr,
- "ERROR: available nics of %d exceeds array limit of %d\n",
- info.nics_available, MX_MAX_BOARDS);
- exit(1);
- }
- ret = mx_get_info(0, MX_NATIVE_REQUESTS, 0, 0, &info.max_requests,sizeof(info.max_requests));
- if (ret != MX_SUCCESS)
- {
- fprintf(stderr, "Failed to do mx_get_info: %s\n", mx_strerror(ret));
- exit(1);
- }
- fprintf(stderr, "MX supports %d native requests\n", info.max_requests);
-
- mx_ep_opened = 1;
- return info;
-}
-
-void close_mx() { mx_finalize(); }
-
-int64_t wait_for_first_system(const int next_cycle, const int64_t prev_sec_and_cycle)
-{
- int wait_count = 0;
- int64_t expected_sec_and_cycle = prev_sec_and_cycle;
- int64_t result = 0;
- receive_map_entry_t cur_row[MXR_MAX_DCU];
-
- if (prev_sec_and_cycle == 0)
- {
- expected_sec_and_cycle = 0;
- }
- bool keep_waiting = true;
- do
- {
- receive_map_copy_row(next_cycle, cur_row);
- for (int ii = 0; ii < MXR_MAX_DCU; ++ii)
- {
- if (cur_row[ii].gps_sec_and_cycle >= expected_sec_and_cycle
- && cur_row[ii].gps_sec_and_cycle != 0
- && extract_cycle(cur_row[ii].gps_sec_and_cycle) == next_cycle)
- {
- result = cur_row[ii].gps_sec_and_cycle;
- keep_waiting = false;
- break;
- }
- }
- if (keep_waiting)
- {
- usleep(1000);
- ++wait_count;
- }
- if (exit_main_loop.is_set())
- {
- return 0;
- }
- } while (keep_waiting);
-
- return result;
-}
-
-void consentrate_data(const int64_t expected_sec_and_cycle, volatile daq_multi_dcu_data_t *dest, unsigned int dest_size,
- scoreboard_manager& manager)
-{
- receive_map_entry_t cur_row[MXR_MAX_DCU];
-
- int cur_cycle = static_cast<int>(extract_cycle(expected_sec_and_cycle));
- buffer_index_t cycle_index = cycle_to_buffer_index(cur_cycle);
-
- volatile char* dest_data = &(dest->dataBlock[0]);
- dest->header.fullDataBlockSize = 0;
- dest->header.dcuTotalModels = 0;
- unsigned int remaining = dest_size - sizeof(daq_multi_dcu_header_t);
-
- //receive_map_copy_row(cur_cycle, cur_row);
- //receive_map_copy( local_map );
- //std::copy(&(local_map[cycle_index][0]), &(local_map[cycle_index][MXR_MAX_DCU - 1]), cur_row);
- manager.get_scoreboard_row(cur_cycle, cur_row);
-
- int model_index = 0;
-
- for (int ii = 0; ii < MXR_MAX_DCU; ++ii)
- {
- if (cur_row[ii].gps_sec_and_cycle == expected_sec_and_cycle)
- {
- int data_size = 0;
- single_dcu_block& cur_block = mxr_data[dcu_index_t(ii)][cycle_index];
- {
- lock_guard<single_dcu_block> l_(cur_block);
- //dest->header.dcuheader[model_index] = cur_block.header;
- memcpy((void*)&(dest->header.dcuheader[model_index]), &(cur_block.header), sizeof(cur_block.header));
- data_size = cur_block.header.dataBlockSize + cur_block.header.tpBlockSize;
- if (data_size > remaining)
- {
- data_size = remaining;
- }
- //std::copy(cur_block.data, cur_block.data + data_size, dest_data);
- memcpy((char*)dest_data, cur_block.data, static_cast<size_t>(data_size));
- }
- dest->header.fullDataBlockSize += data_size;
- dest_data += data_size;
- remaining -= data_size;
- ++model_index;
- manager.mark_received(ii);
- }
- }
- dest->header.dcuTotalModels = model_index;
-
-}
-
-void dcu_mask_difference_printer(const bool prev_flag, const bool cur_flag, unsigned int index)
-{
- std::cout << (prev_flag ? "(-" : "(+") << index << ") ";
-};
-
-void dump_dcu_mask_diffs(const dcu_mask_t& prev, const dcu_mask_t& cur)
-{
- if (prev == cur)
- {
- return;
- }
-
- std::cout << "\nDCU state change:\n";
- foreach_difference(prev, cur, dcu_mask_difference_printer);
- std::cout << "\n";
-}
-
-void show_help(char *prog)
-{
- std::cout << "Usage " << prog << " options\n\nWhere options are:\n";
- std::cout << "\t-h - Show this help\n";
- std::cout << "\t-b <name> - Set the output mbuf name - defaults to " << DEFAULT_BUFFER_NAME << "\n";
- std::cout << "\t-m <size> - Set the output mbuf size in MB [20-100] - defaults to " << DEFAULT_MAX_DATA_SIZE_MB << "\n";
- std::cout << "\t-g <num> - Set the dolphin group number to transmit data on.\n";
- std::cout << "\t-X <path> - Debug dump of the receive timing map to the given path when a dcu is missed.\n";
- std::cout << std::endl;
-}
-
-// *************************************************************************
-// Catch Control C to end cod in controlled manner
-// *************************************************************************
-void intHandler(int dummy) {
- exit_main_loop.set();
-}
-
-void sigpipeHandler(int dummy)
-{}
-
-
-options_t parse_options(int argc, char *const *argv) {
- options_t opts;
- int arg = 0;
- while ((arg = getopt(argc, argv, "b:g:hm:X:")) != EOF)
- {
- switch (arg) {
- case 'b':
- opts.buffer_name = optarg;
- break;
- case 'm':
- opts.max_data_size_mb = static_cast<unsigned int>(atoi(optarg));
- if (opts.max_data_size_mb < 20)
- {
- std::cerr << "Min data block size is 20 MB\n";
- opts.abort = true;
- }
- else if (opts.max_data_size_mb > 100)
- {
- std::cerr << "Max data block size is 100 MB\n";
- opts.abort = true;
- }
- break;
- case 'g':
- opts.dolphin_group = static_cast<unsigned int>(atoi(optarg));
- opts.xmit_data = true;
- break;
- case 'X':
- opts.receive_map_dump_fname = optarg;
- break;
- case 'h':
- default:
- show_help(argv[0]);
- opts.abort = true;
- }
- }
- return opts;
-}
-
-int main(int argc, char *argv[]) {
-
- int max_data_size = 0;
- volatile daq_multi_cycle_header_t* ifo_header = NULL;
- volatile char* ifo_data = NULL;
- std::fstream scoreboard_dump_stream;
- scoped_ptr<simple_dolphin> dolphin;
-
- daq_multi_cycle_header_t *xmitHeader[IX_BLOCK_COUNT];
- static int xmitDataOffset[IX_BLOCK_COUNT];
- int xmitBlockNum = 0;
-
- options_t opts = parse_options(argc, argv);
- if (opts.abort)
- {
- exit(1);
- }
- max_data_size = opts.max_data_size_mb * 1024*1024;
- if (!opts.receive_map_dump_fname.empty())
- {
- scoreboard_dump_stream.open(opts.receive_map_dump_fname.c_str(), std::ios::out | std::ios::trunc);
- scoreboard_dump_stream << "Opening dump stream for output" << std::endl;
- std::cout << "Opening dump stream for output '" << opts.receive_map_dump_fname << "'" << std::endl;
- if (!scoreboard_dump_stream.is_open())
- {
- std::cerr << "Unable to open debug stream" << std::endl;
- exit(1);
- }
- }
-
- ifo_header = reinterpret_cast<volatile daq_multi_cycle_header_t *>(findSharedMemorySize(opts.buffer_name.c_str(), opts.max_data_size_mb));
- ifo_data = (volatile char*)ifo_header + sizeof(daq_multi_cycle_header_t);
- unsigned int cycle_data_size = (max_data_size - sizeof(daq_multi_cycle_header_t)) / DAQ_NUM_DATA_BLOCKS_PER_SECOND;
- cycle_data_size -= (cycle_data_size % 8);
- std::cout << "cycle data size = " << cycle_data_size << "\t" << opts.max_data_size_mb << "\n";
- ifo_header->cycleDataSize = cycle_data_size;
- ifo_header->maxCycle = DAQ_NUM_DATA_BLOCKS;
- ifo_header->curCycle = DAQ_NUM_DATA_BLOCKS + 1;
-
- if (opts.xmit_data)
- {
- segmentId = opts.dolphin_group;
- dolphin.reset(new simple_dolphin());
-
- char* cur_write_address = (char*)writeAddr;
- for (int ii = 0; ii < IX_BLOCK_COUNT; ++ii)
- {
- xmitHeader[ii] = (daq_multi_cycle_header_t*)cur_write_address;
- cur_write_address += IX_BLOCK_SIZE;
- xmitDataOffset[ii] = IX_BLOCK_SIZE * ii + sizeof(struct daq_multi_cycle_header_t);
- }
- }
-
- // set up to catch Control C
- signal(SIGINT,intHandler);
- // setup to ignore sig pipe
- signal(SIGPIPE, sigpipeHandler);
-
- local_mx_info mx_info = open_mx();
-
- int bp_aray[MX_MAX_BOARDS][MX_MAX_ENDPOINTS];
- pthread_t gm_tid[MX_MAX_BOARDS][MX_MAX_ENDPOINTS];
-
- std::cout << "&bp_aray[0][0] = " << &bp_aray[0][0] << "\n";
- std::cout << "&bp_aray[1][0] = " << &bp_aray[1][0] << "\n";
-
- for (int bnum = 0; bnum < mx_info.nics_available; bnum++) { // Start
- for (int j = 0; j < mx_info.max_endpoints; j++) {
- int bp = j;
- bp = bp + (bnum * 256);
- /* calculate address within array */
- bp_aray[bnum][j] = bp;
- //void *bpPtr = (int *)(bp_aray + bnum) + j;
- void *bpPtr = (void *)(&bp_aray[bnum][j]);
- int my_err_no = 0;
-
- std::cout << "setup for board " << bnum << " ep " << j << " *bpPtr = " << *(int*)bpPtr << " bpPtr = " << bpPtr << "\n";
-
- my_err_no =
- pthread_create(&gm_tid[bnum][j], NULL, receiver_thread, bpPtr);
- if (my_err_no)
- {
- char errmsgbuf[80];
- strerror_r(my_err_no, errmsgbuf, sizeof(errmsgbuf));
- fprintf(stderr, "pthread_create() err=%s", errmsgbuf);
- exit(1);
- }
- }
- }
- int next_cycle = 0;
- int64_t prev_sec_and_cycle = 0;
- dcu_mask_t receive_mask;
- dcu_mask_t prev_receive_mask;
- receive_map_t local_map;
-
- std::cout << "Threads created, entering main loop\n";
- int dump_scoreboard_for_cycles = 0;
-
- do
- {
- scoreboard_manager scoreboard(local_map, receive_mask);
- int64_t cur_sec_and_cycle = wait_for_first_system(next_cycle, prev_sec_and_cycle);
-
- if (cur_sec_and_cycle != 0)
- {
- if (prev_sec_and_cycle > 0)
- {
- receive_mask.clear();
-
- int64_t sending_cycle = extract_cycle(prev_sec_and_cycle);
- volatile daq_multi_dcu_data_t *ifoDataBlock = reinterpret_cast<volatile daq_multi_dcu_data_t *>(
- ifo_data + (cycle_data_size * sending_cycle));
- consentrate_data(prev_sec_and_cycle, ifoDataBlock, cycle_data_size, scoreboard);
- ifo_header->curCycle = static_cast<unsigned int>(sending_cycle);
-
- if (opts.xmit_data)
- {
- sci_error_t err = SCI_ERR_OK;
- xmitBlockNum = ifo_header->curCycle % IX_BLOCK_COUNT;
- SCIMemCpy(sequence, (void*)ifoDataBlock, remoteMap, xmitDataOffset[xmitBlockNum], cycle_data_size, memcpyFlag, &err);
- xmitHeader[xmitBlockNum]->maxCycle = ifo_header->maxCycle;
- xmitHeader[xmitBlockNum]->cycleDataSize = cycle_data_size;
- xmitHeader[xmitBlockNum]->curCycle = ifo_header->curCycle;
- SCIFlush(sequence, SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-
- missing_map_t missing_map;
- foreach_difference(prev_receive_mask, receive_mask, missing_map);
- if (missing_map.differences > 0)
- {
- dump_scoreboard_for_cycles = 5;
- }
-
- dump_dcu_mask_diffs(prev_receive_mask, receive_mask);
- prev_receive_mask = receive_mask;
-
- if (dump_scoreboard_for_cycles > 0 && !opts.receive_map_dump_fname.empty())
- {
- --dump_scoreboard_for_cycles;
- dump_scoreboard(scoreboard_dump_stream, local_map, missing_map, prev_sec_and_cycle);
-// if (dump_scoreboard_for_cycles == 1)
-// {
-// //exit_main_loop.set();
-// }
- }
- }
- prev_sec_and_cycle = cur_sec_and_cycle;
- ++next_cycle;
- next_cycle %= 16;
- }
- } while (!exit_main_loop.is_set());
-
- std::cout << "stopping threads\n";
- threads_should_stop.set();
- sleep(2);
- close_mx();
- return 0;
-}
diff --git a/src/ix_stream/mx_rcv_simple.cc b/src/ix_stream/mx_rcv_simple.cc
deleted file mode 100644
index 3be579e11..000000000
--- a/src/ix_stream/mx_rcv_simple.cc
+++ /dev/null
@@ -1,631 +0,0 @@
-// Myrinet MX DQ data receiver
-//#define MX_THREAD_SAFE 1
-//#include "config.h"
-#include "myriexpress.h"
-#include "mx_extensions.h"
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <stdio.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include "../include/daqmap.h"
-//#include "debug.h"
-//#include "daqd.hh"
-//#include "mx_rcvr.hh"
-#include "gps.hh"
-
-const int DEFAULT_THREAD_SAFE_VALUE = true;
-const int DEFAULT_CLOCK_OFFSET_VALUE = 0;
-const int DEFAULT_CYCLE_DELAY_VALUE = 0;
-const int DEFAULT_TRAIL_CYCLE_MS_VALUE = 20;
-
-/***
- * A simple atomic counter that should be implemented lock free.
- * This uses compiler builtins that clang & gcc recognize.
- * This is suitable for sharing between threads.
- */
-template <typename T>
-class atomic_counter {
-public:
- typedef T value_type;
-
- atomic_counter(): val_(0) {}
-
- void increment()
- {
- __sync_add_and_fetch(&val_, 1);
- }
- value_type get()
- {
- return __sync_add_and_fetch(&val_, 0);
- }
- void clear()
- {
- __sync_and_and_fetch(&val_, 0);
- }
-private:
- atomic_counter(const atomic_counter& other);
- atomic_counter& operator=(const atomic_counter& other);
-
- T val_;
-};
-
-
-template <typename T>
-class simple_guard {
-public:
- explicit simple_guard(T& lock): lock_(lock) { lock_.lock(); }
- ~simple_guard() { lock_.unlock(); }
-private:
- simple_guard(const simple_guard& other);
- simple_guard operator=(const simple_guard& other);
- T& lock_;
-};
-
-template <typename T>
-class noop_guard {
-public:
- explicit noop_guard(T& t) {}
-};
-
-class base_lock {
-public:
- base_lock(): l_()
- {
- pthread_spin_init(&l_, PTHREAD_PROCESS_PRIVATE);
- }
- ~base_lock()
- {
- pthread_spin_destroy(&l_);
- }
- void lock()
- {
- pthread_spin_lock(&l_);
- }
- void unlock()
- {
- pthread_spin_unlock(&l_);
- }
-private:
- base_lock(const base_lock& other);
- base_lock operator=(const base_lock& other);
- pthread_spinlock_t l_;
-};
-
-/// Define max boards, endpoints for mx_rcvr
-#define MX_MAX_BOARDS 4
-#define MX_MAX_ENDPOINTS 32
-
-//struct rmIpcStr gmDaqIpc[DCU_COUNT];
-//struct cdsDaqNetGdsTpNum gdsTpNumSpace[2][DCU_COUNT];
-//struct cdsDaqNetGdsTpNum * gdsTpNum[2][DCU_COUNT];
-void *directed_receive_buffer[DCU_COUNT];
-int _log_level = 10;
-
-#define FILTER 0x12345
-#define MATCH_VAL 0xabcdef
-#define DFLT_EID 1
-#define DFLT_LEN 8192
-#define DFLT_END 128
-#define MAX_LEN (1024*1024*1024)
-#define DFLT_ITER 1000
-#define NUM_RREQ 256
-#define NUM_SREQ 4 /* currently constrained by MX_MCP_RDMA_HANDLES_CNT*/
-
-#define MATCH_VAL_MAIN (1 << 31)
-#define MATCH_VAL_THREAD 1
-
-struct daqMXdata {
- struct rmIpcStr mxIpcData;
- cdsDaqNetGdsTpNum mxTpTable;
- char mxDataBlock[DAQ_DCU_BLOCK_SIZE];
-};
-
-struct MXdata {
- MXdata(): data(), lock_() { memset((void*)&data, 0, sizeof(data)); }
-
- daqMXdata data;
-
- void lock() { lock_.lock(); }
- void unlock() { lock_.unlock(); }
-private:
- base_lock lock_;
-};
-
-typedef simple_guard<MXdata> MXLockGuard;
-typedef noop_guard<MXdata> MXNoopGuard;
-MXdata received_data[DCU_COUNT];
-atomic_counter<long> mx_msg_counter;
-
-static const int buf_size = DAQ_DCU_BLOCK_SIZE * 2;
-
-template <typename LockGuardT>
-void
-receiver_mx(int neid)
-{
- mx_status_t stat;
- mx_segment_t seg;
- uint32_t result;
- struct timeval start_time, end_time;
- char *buffer;
- int myErrorStat = 0;
- int copySize;
- //float *testData;
- uint32_t match_val=MATCH_VAL_MAIN;
- mx_request_t req[NUM_RREQ];
- mx_endpoint_t ep[MX_MAX_BOARDS*MX_MAX_ENDPOINTS];
-
- // Set thread parameters
- char my_thr_label[16];
- snprintf(my_thr_label,16,"dqmx%03x",neid);
- char my_thr_name[40];
- snprintf(my_thr_name,40,"mx receiver %03x",neid);
- //daqd_c::set_thread_priority(my_thr_name, my_thr_label, MX_THREAD_PRIORITY,0);
-
- uint32_t board_num = (neid >> 8);
- uint32_t ep_num = neid & 0xff;
-
- mx_return_t ret = mx_open_endpoint(board_num, ep_num, FILTER, NULL, 0, &ep[neid]);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to open board %d endpoint %d error %s\n", board_num, ep_num, mx_strerror(ret));
- exit(1);
- }
-
- // Allocate NUM_RREQ MX packet receive buffers
- int len = sizeof(struct daqMXdata);
- buffer = (char *)malloc(len * NUM_RREQ);
- if (buffer == NULL) {
- fprintf(stderr, "Can't allocate buffers here\n");
- exit(1);
- }
-
-
- /* pre-post our receives */
- for (int cur_req = 0; cur_req < NUM_RREQ; cur_req++) {
- seg.segment_ptr = &buffer[cur_req * len];
- seg.segment_length = len;
- mx_endpoint_t ep1 = ep[neid];
- mx_irecv(ep1, &seg, 1, match_val, MX_MATCH_MASK_NONE, 0,
- &req[cur_req]);
- }
-
- mx_set_error_handler(MX_ERRORS_RETURN);
- gettimeofday(&start_time, NULL);
- int kk = 0;
- do {
- for (int count = 0; count < NUM_RREQ; count++) {
- /* Wait for the receive to complete */
- int cur_req = count & (NUM_RREQ - 1);
-
-// mx_test_or_wait(blocking, ep, &sreq, MX_INFINITE, &stat, &result);
- again:
- mx_return_t ret = mx_wait(ep[neid], &req[cur_req], MX_INFINITE, &stat, &result);
- myErrorStat = 0;
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "mx_cancel() eid=%03x failed with status %s\n", neid, mx_strerror(ret));
- exit(1); // Not clear what to do in this case; this indicates shortage of memory or resources
- }
- if (result == 0) { // Request incomplete
- goto again; // Restart incomplete request
- } else { // Request is complete
- if (stat.code != MX_STATUS_SUCCESS) { // Request completed, but bad code
- fprintf(stderr, "mx_wait failed in rcvr eid=%03x, reqn=%d; wait did not complete; status code is %s\n",
- neid, count, mx_strstatus(stat.code));
- //mx_return_t ret = mx_cancel(ep[neid], &req[cur_req], &result);
- mx_endpoint_addr_t ep_addr;
- ret = mx_get_endpoint_addr(ep[neid], &ep_addr);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "mx_get_endpoint_addr() eid=%03x failed with status %s\n", neid, mx_strerror(ret));
- } else {
- ret = mx_disconnect(ep[neid], ep_addr);
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "mx_disconnect() eid=%03x failed with status %s\n", neid, mx_strerror(ret));
- } else {
- fprintf(stderr, "disconnected from the sender on endpoint %03x\n", neid);
- }
- }
- myErrorStat = 1;
- }
- }
- // Fall through if the request is complete and the code is success
- //
-
- copySize = stat.xfer_length;
- // seg.segment_ptr = buffer[cur_req];
- seg.segment_ptr = &buffer[cur_req * len];
- seg.segment_length = len;
-
- if (!myErrorStat) {
- //printf("received one\n");
- struct daqMXdata *dataPtr = (struct daqMXdata *) seg.segment_ptr;
- int dcu_id = dataPtr->mxIpcData.dcuId;
-
- if (dcu_id < 0 || dcu_id > (DCU_COUNT-1)) {
- mx_irecv(ep[neid], &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- continue; // Bad DCU ID
- }
-// if (daqd.dcuSize[0][dcu_id] == 0) {
-// mx_irecv(ep[neid], &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
-// continue; // Unconfigured DCU
-// }
-
-// int cycle = dataPtr->mxIpcData.cycle;
-// int len = dataPtr->mxIpcData.dataBlockSize;
-//
- {
- LockGuardT g_(received_data[dcu_id]);
- received_data[dcu_id].data = *dataPtr;
- }
- mx_msg_counter.increment();
-// if (dcu_id == 7)
-// {
-// int cycle = dataPtr->mxIpcData.cycle;
-// fprintf(stderr, "rcv >>> id(%d) %d bp: %d %d:%d (%d %d bytes)\n",
-// (int)dcu_id,
-// (int)(dataPtr->mxIpcData.cycle),
-// (int)(dataPtr->mxIpcData.bp[cycle].cycle),
-// (int)(dataPtr->mxIpcData.bp[cycle].timeSec),
-// (int)(dataPtr->mxIpcData.bp[cycle].timeNSec),
-// (int)copySize,
-// (int)len
-// );
-// }
-
-// char *dataDest = (char *)((char *)(directed_receive_buffer[dcu_id]) + buf_size * cycle);
-//
-// // Move the block data into the buffer
-// memcpy (dataDest, dataSource, len);
-//
-// // Assign IPC data
-// gmDaqIpc[dcu_id].crc = dataPtr->mxIpcData.crc;
-// gmDaqIpc[dcu_id].dcuId = dataPtr->mxIpcData.dcuId;
-// gmDaqIpc[dcu_id].bp[cycle].timeSec = dataPtr->mxIpcData.bp[cycle].timeSec;
-// gmDaqIpc[dcu_id].bp[cycle].crc = dataPtr->mxIpcData.bp[cycle].crc;
-// gmDaqIpc[dcu_id].bp[cycle].cycle = dataPtr->mxIpcData.bp[cycle].cycle;
-// gmDaqIpc[dcu_id].dataBlockSize = dataPtr->mxIpcData.dataBlockSize;
-//
-// fprintf(stderr, "dcu %d gps %d %d %d\n", dcu_id, dataPtr->mxIpcData.bp[cycle].timeSec, cycle, dataPtr->mxIpcData.bp[cycle].timeNSec);
-//
-// // Assign test points table
-// //*gdsTpNum[0][dcu_id] = dataPtr->mxTpTable;
-//
-// gmDaqIpc[dcu_id].cycle = cycle;
-//#ifndef USE_MAIN
-//#ifndef USE_SYMMETRICOM
-// if (daqd.controller_dcu == dcu_id) {
-// controller_cycle = cycle;
-// DEBUG(6, printf("Timing dcu=%d cycle=%d\n", dcu_id, controller_cycle));
-// }
-//#endif
-//#endif
-// daqd.producer1.rcvr_stats[dcu_id].tick();
- }
- //daqd.producer1.rcvr_stats[neid].tick();
- mx_irecv(ep[neid], &seg, 1, match_val, MX_MATCH_MASK_NONE, 0, &req[cur_req]);
- }
- } while(1);
- gettimeofday(&end_time, NULL);
- fprintf(stderr, "mx_wait failed in rcvr after do loop\n");
- free(buffer);
- //exit(1);
-}
-
-void *
-receiver_thread_safe(void *args)
-{
- receiver_mx<MXLockGuard>(*reinterpret_cast<int*>(args));
- return NULL;
-}
-
-void *
-receiver_UNSAFE(void *args)
-{
- receiver_mx<MXNoopGuard>(*reinterpret_cast<int*>(args));
- return NULL;
-}
-
-int mx_ep_opened = 0;
-
-struct mx_info {
- uint32_t max_endpoints;
- uint32_t nics_available;
-};
-
-/// Initialize MX library.
-/// Returns the maximum number of end-points supported in the system.
-mx_info
-open_mx(void) {
- mx_info results;
- uint16_t my_eid;
- uint32_t board_id;
- uint32_t filter;
- char *sysname;
- int c;
- extern char *optarg;
- mx_return_t ret;
- int fd;
- static uint32_t max_endpoints = 0;
- static uint32_t nics_available = 0;
-
- static char omx_errors_env[] = "OMX_ERRORS_ARE_FATAL=0";
- static char mx_errors_env[] = "MX_ERRORS_ARE_FATAL=0";
-
- if (!mx_ep_opened) {
- printf("%ld\n", sizeof(struct daqMXdata));
-
- // So that openmx is not aborting on connection loss
- putenv(omx_errors_env);
- putenv(mx_errors_env);
-
- mx_init();
-
- /* set up defaults */
- sysname = NULL;
- filter = FILTER;
- ret = mx_get_info(0, MX_MAX_NATIVE_ENDPOINTS, 0, 0, &max_endpoints, sizeof(max_endpoints));
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to do mx_get_info: %s\n", mx_strerror(ret));
- exit(1);
- }
- fprintf(stderr, "MX has %d maximum end-points configured\n", max_endpoints);
- ret = mx_get_info(0, MX_NIC_COUNT, 0, 0, &nics_available, sizeof(nics_available));
- if (ret != MX_SUCCESS) {
- fprintf(stderr, "Failed to do mx_get_info: %s\n", mx_strerror(ret));
- exit(1);
- }
- fprintf(stderr, "%d MX NICs available\n", nics_available);
- /// make sure these don't exceed array allocations
- if (max_endpoints > MX_MAX_ENDPOINTS) {
- fprintf(stderr, "ERROR: max end-points of %d exceeds array limit of %d\n", max_endpoints, MX_MAX_ENDPOINTS);
- exit(1);
- }
- if (nics_available > MX_MAX_BOARDS) {
- fprintf(stderr, "ERROR: available nics of %d exceeds array limit of %d\n", nics_available, MX_MAX_BOARDS);
- exit(1);
- }
-
- mx_ep_opened = 1;
- }
- results.nics_available = nics_available;
- results.max_endpoints = max_endpoints;
- return results;
-}
-
-void short_sleep(unsigned long sleep_ms = 10)
-{
- struct timespec wait = {0, 0};
- wait.tv_nsec = 1000000UL * sleep_ms;
- nanosleep(&wait, NULL);
-}
-
-void wait_for_mx_messages()
-{
- while (mx_msg_counter.get() == 0)
- {
- short_sleep();
- }
-}
-
-GPS::gps_time
-wait_for_end_of_cycle(GPS::gps_clock& clock, int cycle, int cycle_delay) {
- const unsigned int c = 1000000000 / 16;
-
- int target_cycle = (cycle + cycle_delay + 1) % 16;
-
- long lower_nano = target_cycle * c;
- long upper_nano = lower_nano + c;
-
- //fprintf(stderr, "wfeoc %d %d - waiting for %d %ld %ld\n", cycle, cycle_delay, target_cycle, lower_nano, upper_nano);
-
- GPS::gps_time now = clock.now();
- while (now.nanosec < lower_nano || now.nanosec > upper_nano)
- {
- short_sleep();
- now = clock.now();
- }
- return now;
-}
-
-void close_mx() {
- mx_finalize();
-}
-
-template <typename LockGuardT>
-void
-concentrate_loop(GPS::gps_clock& clock, int cycle_delay, int trail_cycle_ms)
-{
- int cur_cycle = 0;
- bool timing_glitch[DCU_COUNT];
-
- for (int i = 0; i < DCU_COUNT; ++i)
- {
- timing_glitch[i] = true;
- }
-
- std::cout << "Waiting for MX messages to start ...\n";
- wait_for_mx_messages();
- std::cout << "MX messages have been received, will concentrate data at the start of the next second\n";
-
- for (int cur_cycle = 0;; cur_cycle = (cur_cycle + 1) % 16)
- {
- GPS::gps_time now = wait_for_end_of_cycle(clock, cur_cycle, cycle_delay);
- short_sleep(trail_cycle_ms);
- long expected_gps = now.sec;
- if (cur_cycle >= (15 - cycle_delay))
- {
- --expected_gps;
- }
- bool need_nl = false;
- for (int cur_dcu = 0; cur_dcu < DCU_COUNT; ++cur_dcu)
- {
- unsigned int timeSec = 0;
- unsigned int timeNSec = 0;
- unsigned int cycle =0 ;
- {
- LockGuardT g_(received_data[cur_dcu]);
- cycle = received_data[cur_dcu].data.mxIpcData.cycle;
- timeSec = received_data[cur_dcu].data.mxIpcData.bp[cur_cycle].timeSec;
- timeNSec = received_data[cur_dcu].data.mxIpcData.bp[cur_cycle].timeNSec;
- }
-// if (cur_dcu == 7)
-// {
-// std::cout << "now (" << now.sec << ":" << now.nanosec << ") cycle: " << cur_cycle;
-// std::cout << " received cycle - " << cycle << " " << timeSec << ":" << timeNSec << " exp: " << expected_gps << std::endl;
-// }
- if (expected_gps == timeSec)
- {
- if (timing_glitch[cur_dcu])
- {
- std::cout << " +(" << cur_dcu << ")";
- need_nl = true;
- }
- timing_glitch[cur_dcu] = false;
- }
- else
- {
- if (!timing_glitch[cur_dcu])
- {
- std::cout << " -(" << cur_dcu << ")";
- need_nl = true;
- }
- timing_glitch[cur_dcu] = true;
- }
- }
- if (need_nl)
- {
- std::cout << "\n";
- }
- }
-}
-
-struct options_t
-{
- options_t(): thread_safe(DEFAULT_THREAD_SAFE_VALUE),
- clock_offset(DEFAULT_CLOCK_OFFSET_VALUE),
- cycle_delay(DEFAULT_CYCLE_DELAY_VALUE),
- trail_cycle_ms(DEFAULT_TRAIL_CYCLE_MS_VALUE),
- abort(false) {}
-
- bool thread_safe;
- int clock_offset;
- int cycle_delay;
- int trail_cycle_ms;
- bool abort;
-};
-
-void show_help(const char* prog)
-{
- std::cout << "Usage:" << prog << " options\n\nWhere options are:\n";
- std::cout << "\t-h - Show this help\n";
- std::cout << "\t-S - Use the safe thread code [Default]\n";
- std::cout << "\t-U - Use the UNSAFE thread code (match the daqd code)\n";
- std::cout << "\t-G <offset> - Offset to apply to the gps clock [Default=" << DEFAULT_CLOCK_OFFSET_VALUE << "]\n";
- std::cout << "\t-c <cyle delay> - Cycle delay value [Default=" << DEFAULT_CYCLE_DELAY_VALUE << "]\n";
- std::cout << "\t-t <ms> - Wait for the given ms after each cycle to start concentrating the data [Default=" << DEFAULT_TRAIL_CYCLE_MS_VALUE << "]\n";
- std::cout << std::endl;
-}
-
-options_t parse_options(int argc, char *const *argv) {
- options_t opts;
- int arg = 0;
- while ((arg = getopt(argc, argv, "SUG:c:t:h")) != EOF)
- {
- switch (arg) {
- case 'S':
- opts.thread_safe = 1;
- break;
- case 'U':
- opts.thread_safe = 0;
- break;
- case 'G':
- opts.clock_offset = atoi(optarg);
- break;
- case 'c':
- opts.cycle_delay = atoi(optarg);
- break;
- case 't':
- opts.trail_cycle_ms = atoi(optarg);
- break;
- case 'h':
- default:
- opts.abort = true;
- }
- }
- if (opts.abort)
- {
- show_help(argv[0]);
- }
- return opts;
-}
-
-int main(int argc, char* argv[]) {
- options_t opts = parse_options(argc, argv);
-
- if (opts.abort)
- {
- exit(1);
- }
- GPS::gps_clock clock(opts.clock_offset);
- if (!clock.ok())
- {
- fprintf(stderr, "This program requires a LIGO gps time device to be present.");
- exit(1);
- }
-
-
- // Allocate receive buffers for each configured DCU
- for (int i = 0; i < DCU_COUNT; i++) {
- //if (0 == daqd.dcuSize[0][i]) continue;
-
- directed_receive_buffer[i] = malloc(2*DAQ_DCU_BLOCK_SIZE*DAQ_NUM_DATA_BLOCKS);
- if (directed_receive_buffer[i] == 0) {
- fprintf (stderr, "[MX recv] Couldn't allocate recv buffer\n");
- exit(1);
- }
- }
- mx_info info = open_mx();
-
- int bp_aray[MX_MAX_BOARDS][MX_MAX_ENDPOINTS];
- pthread_t gm_tid[MX_MAX_BOARDS][MX_MAX_ENDPOINTS];
-
- for (int bnum = 0; bnum < info.nics_available; bnum++) { // Start
- for (int j = 0; j < info.max_endpoints; j++) {
- int bp = j;
- bp = bp + (bnum * 256);
- /* calculate address within array */
- bp_aray[bnum][j] = bp;
- //void *bpPtr = (int *)(bp_aray + bnum) + j;
- void *bpPtr = (void *)(&bp_aray[bnum][j]);
- int my_err_no = 0;
-
- std::cout << "setup for board " << bnum << " ep " << j << " *bpPtr = " << *(int*)bpPtr << " bpPtr = " << bpPtr << "\n";
-
- my_err_no =
- pthread_create(&gm_tid[bnum][j], NULL, (opts.thread_safe ? receiver_thread_safe : receiver_UNSAFE), bpPtr);
- if (my_err_no)
- {
- char errmsgbuf[80];
- strerror_r(my_err_no, errmsgbuf, sizeof(errmsgbuf));
- fprintf(stderr, "pthread_create() err=%s", errmsgbuf);
- exit(1);
- }
- }
- }
- if (opts.thread_safe)
- {
- concentrate_loop<MXLockGuard>(clock, opts.cycle_delay, opts.trail_cycle_ms);
- }
- else
- {
- concentrate_loop<MXNoopGuard>(clock, opts.cycle_delay, opts.trail_cycle_ms);
- }
- return 0;
-}
diff --git a/src/ix_stream/zmq_rcv_ix_xmit.c b/src/ix_stream/zmq_rcv_ix_xmit.c
deleted file mode 100644
index 51309de3a..000000000
--- a/src/ix_stream/zmq_rcv_ix_xmit.c
+++ /dev/null
@@ -1,1335 +0,0 @@
-//
-/// @file zmq_rcv_ix_xmitcc
-/// @brief DAQ data concentrator code. Receives data via ZMQ and sends via Dolphin IX..
-//
-
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/file.h>
-#include <stdio.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include "../drv/crc.c"
-#include <zmq.h>
-#include <assert.h>
-#include <time.h>
-#include "../include/daqmap.h"
-#include "../include/daq_core.h"
-#include "dc_utils.h"
-
-#include "sisci_types.h"
-#include "sisci_api.h"
-#include "sisci_error.h"
-#include "sisci_demolib.h"
-#include "testlib.h"
-
-#include "zmq_transport.h"
-#include "simple_pv.h"
-
-#include <stdarg.h>
-
-#define __CDECL
-
-#define DO_HANDSHAKE 0
-
-
-#include "./dolphin_common.c"
-
-#define MIN_DELAY_MS 5
-#define MAX_DELAY_MS 40
-
-#define MAX_FE_COMPUTERS 64
-
-#define MAX_RECEIVE_BUFFERS 4
-
-#define dbgprintf printf
-
-void null_printf(const char *fmt, ...)
-{}
-
-extern void *findSharedMemorySize(char *,int);
-
-int do_verbose = 0;
-int debug_zmq = 0;
-
-struct thread_info {
- int index;
- char *src_iface;
-};
-struct thread_mon_info {
- int index;
- void *ctx;
-};
-
-typedef struct mask_t {
- int data[DCU_COUNT];
-} mask_t;
-
-typedef struct receive_map_entry_t {
- int64_t gps_sec_and_cycle;
- int64_t s_clock;
-} receive_map_entry_t;
-
-/// \brief structure to hold when data has been received
-typedef struct receive_map_t
-{
- receive_map_entry_t data[MAX_RECEIVE_BUFFERS][MAX_FE_COMPUTERS]; /// indexed by cycle then machine
-} receive_map_t;
-
-pthread_spinlock_t receive_map_lock;
-receive_map_t receive_map;
-
-/// \brief given a gps second + a cycle count merge them into one value
-static inline int64_t calc_gps_sec_and_cycle(int64_t gps_sec, int64_t cycle)
-{
- return (gps_sec << 4) | (0x0f & cycle);
-}
-
-// helper to extract seconds counts out of a combined gps sec + cycle value
-static inline int64_t extract_gps(int64_t gps_sec_and_cycle)
-{
- return gps_sec_and_cycle >> 4;
-}
-
-// helper to extract cycle counts out of a combined gps sec + cycle value
-static inline int64_t extract_cycle(int64_t gps_sec_and_cycle)
-{
- return gps_sec_and_cycle & 0x0f;
-}
-
-
-static inline void mask_clear(mask_t* mask)
-{
- memset(mask->data, 0, sizeof(mask->data));
-}
-
-static inline void mask_set_entry(mask_t* mask, int index)
-{
- const unsigned int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- if (((unsigned int)index) < MAX_INDEX)
- {
- mask->data[index] = 1;
- }
-}
-
-static inline int mask_get_entry(mask_t* mask, int index)
-{
- const unsigned int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- return (((unsigned int)index) < MAX_INDEX ? mask->data[index] : 0);
-}
-
-static inline int mask_diffs(mask_t* input1, mask_t* input2, mask_t* dest)
-{
- const int MAX_INDEX = sizeof(input1->data)/sizeof(input1->data[0]);
- int diffs = 0;
- int i = 0;
- for(; i < MAX_INDEX; ++i)
- {
- diffs += (dest->data[i] = (input1->data[i] != input2->data[i]));
- }
- return diffs;
-}
-
-static inline int mask_differs(mask_t* input1, mask_t* input2)
-{
- const int MAX_INDEX = sizeof(input1->data)/sizeof(input1->data[0]);
- int i = 0;
- for (; i < MAX_INDEX; ++i)
- {
- if (input1->data[i] != input2->data[i])
- {
- return 1;
- }
- }
- return 0;
-}
-
-static inline void mask_dump(mask_t* mask, const char* description)
-{
- printf("%s\n", description);
- const int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- int i = 0;
- for (; i < MAX_INDEX; ++i)
- {
- if (mask->data[i])
- {
- printf("(%d) ", i);
- }
- }
- printf("\n");
-}
-
-static inline void mask_copy(mask_t* dest, const mask_t* src)
-{
- memcpy(dest, src, sizeof(*dest));
-}
-
-static void dump_dcu_mask_diff(mask_t* prev, mask_t* cur, int dcu_endpoint_lookup[DCU_COUNT])
-{
- const int MAX_INDEX = sizeof(cur->data)/sizeof(cur->data[0]);
- if (!mask_differs(prev, cur))
- {
- return;
- }
- printf("\nDCU state change:\n");
- int i = 0;
- for (i = 0; i < MAX_INDEX; ++i)
- {
- if (prev->data[i] < cur->data[i])
- {
- printf("(+%d)e%d ", i, dcu_endpoint_lookup[i]);
- }
- else if (prev->data[i] > cur->data[i])
- {
- printf("(-%d)e%d ", i, dcu_endpoint_lookup[i]);
- }
- }
-}
-
-static void dump_endpoint_mask_diff(mask_t* prev, mask_t* cur)
-{
- const int MAX_INDEX = sizeof(cur->data)/sizeof(cur->data[0]);
- if (!mask_differs(prev, cur))
- {
- return;
- }
- printf("\nEndpoint state change:\n");
- int i = 0;
- for (i = 0; i < MAX_INDEX; ++i)
- {
- if (prev->data[i] < cur->data[i])
- {
- printf("(+%d) ", i);
- }
- else if (prev->data[i] > cur->data[i])
- {
- printf("(-%d) ", i);
- }
- }
- printf("\n");
-}
-
-static void receive_map_init()
-{
- memset(&receive_map, 0, sizeof(receive_map));
- pthread_spin_init(&receive_map_lock, PTHREAD_PROCESS_PRIVATE);
-}
-
-static void receive_map_mark(int fe_index, long gps_sec, int64_t cycle, int64_t clock_val)
-{
- int64_t gps_sec_and_cycle = 0;
-
- int cycle_index = (int)(cycle % MAX_RECEIVE_BUFFERS);
- gps_sec_and_cycle = calc_gps_sec_and_cycle(gps_sec, cycle);
-
- pthread_spin_lock(&receive_map_lock);
- receive_map.data[cycle_index][fe_index].gps_sec_and_cycle = gps_sec_and_cycle;
- receive_map.data[cycle_index][fe_index].s_clock = clock_val;
- pthread_spin_unlock(&receive_map_lock);
-}
-
-static void receive_map_copy_to_local(receive_map_t *dest)
-{
- if (!dest)
- {
- return;
- }
- pthread_spin_lock(&receive_map_lock);
- memcpy(dest, &receive_map, sizeof(receive_map));
- pthread_spin_unlock(&receive_map_lock);
-}
-
-static void dump_recv_mask(FILE *f, int nsys, int64_t gps_sec, int block_index, mask_t* received_mask)
-{
- int fe_index = 0, cycle_index = 0;
- receive_map_t local_map;
- if (!f)
- {
- return;
- }
- receive_map_copy_to_local(&local_map);
- fprintf(f, "----------------------------------------------\n");
- fprintf(f, "-- Gps %ld:%d\n", (long int)gps_sec, (int)block_index);
- fprintf(f, "----------------------------------------------\n");
-
- block_index %= MAX_RECEIVE_BUFFERS;
-
- for (cycle_index = MAX_RECEIVE_BUFFERS - 1; cycle_index >= 0 ; --cycle_index)
- {
- fprintf(f, "% 2d) ", cycle_index);
- for (fe_index = 0; fe_index < nsys; ++fe_index)
- {
- fprintf(f, "|%s%ld:%d (%ld) ",
- (cycle_index != block_index ? " " :
- (mask_get_entry(received_mask, fe_index) ? " " : "*")),
- (long int) local_map.data[cycle_index][fe_index].gps_sec_and_cycle >> 4,
- 0x0f & (int) local_map.data[cycle_index][fe_index].gps_sec_and_cycle,
- (long int) local_map.data[cycle_index][fe_index].s_clock);
- }
- fprintf(f, "|\n");
- }
-
- fprintf(f, "----------------------------------------------\n\n");
-}
-
-
-
-struct thread_info thread_index[DCU_COUNT];
-void *daq_context[DCU_COUNT];
-void *daq_subscriber[DCU_COUNT];
-char *sname[DCU_COUNT]; // Names of FE computers serving DAQ data
-char *local_iface[MAX_FE_COMPUTERS];
-daq_multi_dcu_data_t mxDataBlockSingle[MAX_FE_COMPUTERS][MAX_RECEIVE_BUFFERS];
-const int mc_header_size = sizeof(daq_multi_cycle_header_t);
-int stop_working_threads = 0;
-int start_acq = 0;
-static volatile int keepRunning = 1;
-//int thread_cycle[MAX_FE_COMPUTERS];
-//int thread_timestamp[MAX_FE_COMPUTERS];
-int rcv_errors = 0;
-
-void
-usage()
-{
- fprintf(stderr, "Usage: zmq_rcv_ix_xmit [args] -s server names -m shared memory size -g IX channel \n");
- fprintf(stderr, "-l filename - log file name\n");
- fprintf(stderr, "-b buffername - name of the mbuf to write to\n");
- fprintf(stderr, "-s - server names separated by spaces eg \"x1lsc0 x1susex etc.\"\n");
- fprintf(stderr, "-v - verbose prints diag test data\n");
- fprintf(stderr, "-g - Dolphin IX channel to xmit on\n");
- fprintf(stderr, "-p - Debug pv prefix, requires -P as well\n");
- fprintf(stderr, "-P - Path to a named pipe to send PV debug information to\n");
- fprintf(stderr, "-d - Max delay in milli seconds to wait for a FE to send data, defaults to 10\n");
- fprintf(stderr, "-L ifaces - local interfaces to listen on [used for load balancing], eg \"eth0 eth1\"\n");
- fprintf(stderr, "-z - output zmq debug information (state changes)\n");
- fprintf(stderr, "-X path - output history of received data to given path\n");
- fprintf(stderr, "-h - help\n");
-}
-
-// *************************************************************************
-// Timing Diagnostic Routine
-// *************************************************************************
-static int64_t
-s_clock (void)
-{
-struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-// *************************************************************************
-// Catch Control C to end cod in controlled manner
-// *************************************************************************
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-void sigpipeHandler(int dummy)
-{}
-
-// **********************************************************************************************
-void print_diags(int nsys, int lastCycle, int sendLength, daq_multi_dcu_data_t *ixDataBlock,int dbs[]) {
-// **********************************************************************************************
- int ii = 0;
- // Print diags in verbose mode
- printf("Receive errors = %d\n",rcv_errors);
- printf("Time = %d\t size = %d\n",ixDataBlock->header.dcuheader[0].timeSec,sendLength);
- printf("DCU ID\tCycle \t TimeSec\tTimeNSec\tDataSize\tTPCount\tTPSize\tXmitSize\n");
- for(ii=0;ii<nsys;ii++) {
- printf("%d",ixDataBlock->header.dcuheader[ii].dcuId);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].cycle);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].timeSec);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].timeNSec);
- printf("\t\t%d",ixDataBlock->header.dcuheader[ii].dataBlockSize);
- printf("\t\t%d",ixDataBlock->header.dcuheader[ii].tpCount);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].tpBlockSize);
- printf("\t%d",dbs[ii]);
- printf("\n ");
- }
-}
-
-const char *pevent_zmq(int event, char *buffer, size_t max_size)
-{
- if (!buffer || max_size < 1) {
- return "";
- }
- switch (event) {
- default:
- snprintf(buffer, max_size, "%d", event);
- break;
- case ZMQ_EVENT_CONNECTED:
- strncpy(buffer, "CONNECTED", max_size);
- break;
- case ZMQ_EVENT_CONNECT_DELAYED:
- strncpy(buffer, "CONNECT_DELAYED", max_size);
- break;
- case ZMQ_EVENT_CONNECT_RETRIED:
- strncpy(buffer, "CONNECT_RETRIED", max_size);
- break;
- case ZMQ_EVENT_LISTENING:
- strncpy(buffer, "LISTENING", max_size);
- break;
- case ZMQ_EVENT_BIND_FAILED:
- strncpy(buffer, "BIND_FAILED", max_size);
- break;
- case ZMQ_EVENT_ACCEPTED:
- strncpy(buffer, "ACCEPTED", max_size);
- break;
- case ZMQ_EVENT_ACCEPT_FAILED:
- strncpy(buffer, "ACCEPT_FAILED", max_size);
- break;
- case ZMQ_EVENT_CLOSED:
- strncpy(buffer, "CLOSED", max_size);
- break;
- case ZMQ_EVENT_CLOSE_FAILED:
- strncpy(buffer, "CLOSE_FAILED", max_size);
- break;
- case ZMQ_EVENT_DISCONNECTED:
- strncpy(buffer, "DISCONNECTED", max_size);
- break;
- case ZMQ_EVENT_MONITOR_STOPPED:
- strncpy(buffer, "MONITOR_STOPPED", max_size);
- break;
- }
- buffer[max_size-1] = '\0';
- return buffer;
-}
-
-// *****
-// monitoring thread callback
-// *****
-void *rcvr_thread_mon(void *args)
-{
- int rc;
- char msg_buf[100];
- struct thread_mon_info *info = (struct thread_mon_info*)args;
-
- void *s = zmq_socket(info->ctx, ZMQ_PAIR);
- rc = zmq_connect(s, "inproc://monitor.req");
- while (1)
- {
- zmq_msg_t msg;
- zmq_msg_init(&msg);
- rc = zmq_recvmsg(s, &msg, 0);
- if (rc == -1) break;
- unsigned short *event = (unsigned short*)zmq_msg_data(&msg);
- unsigned int *value = (unsigned int*)(event+1);
- fprintf(stderr, "%s) %d - %s\n", pevent_zmq(*event, msg_buf, sizeof(msg_buf)), *value, sname[info->index]);
- if (zmq_msg_more(&msg)) {
- zmq_msg_init(&msg);
- zmq_msg_recv(&msg, s, 0);
- }
- }
- zmq_close(s);
- free(args);
- return NULL;
-}
-
-// *************************************************************************
-// Thread for receiving DAQ data via ZMQ
-// *************************************************************************
-void *rcvr_thread(void *arg) {
- struct thread_mon_info* mon_info = 0;
- struct thread_info* my_info = (struct thread_info*)arg;
- int mt = my_info->index;
- printf("myarg = %d on iface %s\n", mt, (my_info->src_iface ? my_info->src_iface : "default interface"));
- int cycle = 0;
- daq_multi_dcu_data_t *mxDataBlock;
- char loc[256];
-
- void *zctx = 0;
- void *zsocket = 0;
- int rc = 0;
- pthread_t mon_th_id;
- int buffer_index = 0;
-
-
- zctx = zmq_ctx_new();
- zsocket = zmq_socket(zctx, ZMQ_SUB);
-
- if (debug_zmq) {
- zmq_socket_monitor(zsocket, "inproc://monitor.req", ZMQ_EVENT_ALL);
- mon_info = calloc(1, sizeof(struct thread_mon_info));
- if (!mon_info) {
- fprintf(stderr, "Unable to initialize monitoring thread for %s\n", sname[mt]);
- } else {
- mon_info->index = mt;
- mon_info->ctx = zctx;
- pthread_create(&mon_th_id, NULL, rcvr_thread_mon, mon_info);
- }
- }
-
- rc = zmq_setsockopt(zsocket, ZMQ_SUBSCRIBE, "", 0);
- assert(rc == 0);
- if (!dc_generate_connection_string(loc, sname[mt], sizeof(loc), my_info->src_iface)) {
- fprintf(stderr, "Unable to create connection string for '%s'\n", sname[mt]);
- exit(1);
- }
- dc_set_zmq_options(zsocket);
- rc = zmq_connect(zsocket, loc);
- assert(rc == 0);
-
-
- printf("Starting receive loop for thread %d %s\n", mt, loc);
- char *daqbuffer = (char *)&mxDataBlockSingle[mt];
- mxDataBlock = (daq_multi_dcu_data_t *)daqbuffer;
- do {
- buffer_index = zmq_recv_daq_multi_dcu_t_into_buffer((daq_multi_dcu_data_t*)daqbuffer, zsocket, MAX_RECEIVE_BUFFERS);
-
- // Get the message DAQ cycle number
- cycle = mxDataBlock[buffer_index].header.dcuheader[0].cycle;
- // Pass cycle and timestamp data back to main process
- int64_t cur_time = s_clock();
- receive_map_mark(mt, mxDataBlock[buffer_index].header.dcuheader[0].timeSec, cycle, cur_time);
- //printf("#### thread %d wrote %ld:%d at %ld ####\n", (int)mt, (long int)mxDataBlock[buffer_index].header.dcuheader[0].timeSec, (int)cycle, (long int)cur_time);
- // Run until told to stop by main thread
- } while(!stop_working_threads);
- printf("Stopping thread %d\n",mt);
- usleep(200000);
-
- zmq_close(zsocket);
- zmq_ctx_destroy(zctx);
- return(0);
-
-}
-
-int
-wait_for_data(int nsys, int dataReady[MAX_FE_COMPUTERS], int *nextCycle, int64_t prev_sec_and_cycle, int *rcv_errors)
-{
- int ii = 0;
- int ready_count = 0;
- int timeout = 0;
- int64_t deadline_ms = 0;
- int64_t remaining_time_ms = 1; // this must start non-zero
- int64_t cur_time = 0;
- int next_cycle = *nextCycle;
- int64_t expected_sec_and_cycle = calc_gps_sec_and_cycle(extract_gps(prev_sec_and_cycle + 1), next_cycle);
- receive_map_t data_ready_map;
-
-
- dbgprintf("\n----- wait_for_data nsys=%d nextCycle=%d prev_sec=%ld\n", (int)nsys, (int)*nextCycle, (long int)prev_sec_and_cycle);
-
-
- if (prev_sec_and_cycle == 0)
- {
- expected_sec_and_cycle = 0;
- }
- dbgprintf("expected_sec_and_cycle = %ld\n", (long int)expected_sec_and_cycle);
-
- for (ii = 0; ii < nsys; ++ii)
- {
- dataReady[ii] = 0;
- }
-
- do {
- if (deadline_ms == 0)
- {
- usleep(1000); /* 1ms */
- dbgprintf("<sleep 1ms>");
- }
- else
- {
- int64_t cur_time = s_clock();
- remaining_time_ms = deadline_ms - cur_time;
- if (remaining_time_ms > 0)
- {
- dbgprintf("<sleep 0.2ms - cur_time = %ld remaining = %ldms>", (long int)cur_time, (long int)remaining_time_ms);
- usleep(200); /* 0.2ms */
- }
- }
- receive_map_copy_to_local(&data_ready_map);
-
- for (ii = 0; ii < nsys; ++ii) {
- if (dataReady[ii]) {
- continue;
- }
- // FIXME: what to do if it is a different second?
- // For the first time through expected_sec_and_cycle == 0, so this grabs
- // the first entry.
- // Otherwise we handle the case of ignoring old data (ie a front end dropped out)
- // but what about something that is running fast, or where one FE is signifigantly
- // off on its timing?
- if (data_ready_map.data[next_cycle][ii].gps_sec_and_cycle >= expected_sec_and_cycle)
- {
- dataReady[ii] = 1;
- ++ready_count;
- dbgprintf("receivied system %d at %ld ready_count=%d remaining=%d\n", (int)ii, (long int)data_ready_map.data[next_cycle][ii].s_clock, (int)ready_count, (int)remaining_time_ms);
- if (deadline_ms == 0)
- {
- deadline_ms = data_ready_map.data[next_cycle][ii].s_clock + 55;
- remaining_time_ms = deadline_ms - s_clock();
- dbgprintf("deadline set to %ld\n", (long int)deadline_ms);
- }
- }
- }
- } while (remaining_time_ms > 0 && ready_count != nsys);
- dbgprintf("exiting wait loop, ready_count = %d\nremaining = %d\n---------------\n", (int)ready_count, (int)remaining_time_ms);
-
-// // Wait up to 100ms until received data from at least 1 FE or timeout
-// do {
-// usleep(2000);
-// for(ii=0;ii<nsys;ii++) {
-// if(next_cycle == thread_cycle[ii]) ready_count = 1;
-// }
-// timeout += 1;
-// }while(!ready_count && timeout < 50);
-//
-// // Wait up to delay_ms ms in 1/10ms intervals until data received from everyone or timeout
-// timeout = 0;
-// do {
-// usleep(100);
-// for(ii=0;ii<nsys;ii++) {
-// if(next_cycle == thread_cycle[ii] && !dataReady[ii]) ++ready_count;
-// if(next_cycle == thread_cycle[ii]) dataReady[ii] = 1;
-// }
-// timeout += 1;
-// }while(ready_count < nsys && timeout < delay_cycles);
-// if(timeout >= 100) *rcv_errors += (nsys - ready_count);
- return ready_count;
-}
-
-
-// *************************************************************************
-// Main Process
-// *************************************************************************
-int
-main(int argc, char **argv)
-{
- pthread_t thread_id[MAX_FE_COMPUTERS];
- unsigned int nsys = 1; // The number of mapped shared memories (number of data sources)
- char *sysname;
- char *buffer_name = "ifo";
- int c;
- int ii; // Loop counter
- int delay_ms = 10;
- int delay_cycles = 0;
- unsigned int niface = 0; // The number of local interfaces to split receives across.
- char *local_iface_names = 0;
-
- extern char *optarg; // Needed to get arguments to program
-
- // PV/debug information
- char *pv_prefix = 0;
- char *pv_debug_pipe_name = 0;
- int pv_debug_pipe = -1;
-
- // Declare shared memory data variables
- daq_multi_cycle_header_t *ifo_header;
- char *ifo;
- char *ifo_data;
- int cycle_data_size;
- daq_multi_dcu_data_t *ifoDataBlock;
- char *nextData;
- int max_data_size_mb = 100;
- int max_data_size = 0;
- char *mywriteaddr;
- daq_multi_cycle_header_t *xmitHeader[IX_BLOCK_COUNT];
- static int xmitDataOffset[IX_BLOCK_COUNT];
- int xmitBlockNum = 0;
-
- FILE *recv_map_fp = 0;
- int record_recv_map_for_cycles = 0;
-
- /* set up defaults */
- sysname = NULL;
- int xmitData = 0;
-
- receive_map_init();
-
-
- // Get arguments sent to process
- while ((c = getopt(argc, argv, "b:hs:m:g:vp:P:d:L:zX:")) != EOF) switch(c) {
- case 's':
- sysname = optarg;
- break;
- case 'v':
- do_verbose = 1;
- break;
- case 'z':
- debug_zmq = 1;
- break;
- case 'm':
- max_data_size_mb = atoi(optarg);
- if (max_data_size_mb < 20){
- printf("Min data block size is 20 MB\n");
- return -1;
- }
- if (max_data_size_mb > 100){
- printf("Max data block size is 100 MB\n");
- return -1;
- }
- break;
- case 'g':
- segmentId = atoi(optarg);
- xmitData = 1;
- break;
- case 'b':
- buffer_name = optarg;
- break;
- case 'p':
- pv_prefix = optarg;
- break;
- case 'P':
- pv_debug_pipe_name = optarg;
- break;
- case 'd':
- delay_ms = atoi(optarg);
- if (delay_ms < MIN_DELAY_MS || delay_ms > MAX_DELAY_MS) {
- printf("The delay factor must be between 5ms and 40ms\n");
- return -1;
- }
- break;
- case 'h':
- case 'L':
- local_iface_names = optarg;
- break;
- case 'X':
- recv_map_fp = fopen(optarg, "wt");
- if (!recv_map_fp)
- {
- fprintf(stderr, "Unable to open %s for writting, cannot dump recieve map\n", optarg);
- }
- break;
- default:
- usage();
- exit(1);
- }
- max_data_size = max_data_size_mb * 1024*1024;
- delay_cycles = delay_ms * 10;
-
- if (sysname == NULL) { usage(); exit(1); }
-
- // set up to catch Control C
- signal(SIGINT,intHandler);
- // setup to ignore sig pipe
- signal(SIGPIPE, sigpipeHandler);
-
- printf("Server name: %s\n", sysname);
-
- // Parse names of data servers
- sname[0] = strtok(sysname, " ");
- for(;;) {
- printf("%s\n", sname[nsys - 1]);
- char *s = strtok(0, " ");
- if (!s) break;
- // do not overflow our fixed size buffers
- assert(nsys < DCU_COUNT);
- sname[nsys] = s;
- nsys++;
- }
-
- // Parse local interface names
- if (local_iface_names) {
- local_iface[0] = strtok(local_iface_names, " ");
- niface++;
- for (;;) {
- printf("local interface %s\n", local_iface[niface-1]);
- char *s = strtok(0, " ");
- if (!s) break;
- assert(niface < MAX_FE_COMPUTERS);
- local_iface[niface] = s;
- niface++;
- }
- }
-
- // Get pointers to local DAQ mbuf
- ifo = (char *)findSharedMemorySize(buffer_name,max_data_size_mb);
- ifo_header = (daq_multi_cycle_header_t *)ifo;
- ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
- // Following line breaks daqd for some reason
- // max_data_size *= 1024 * 1024;
- cycle_data_size = (max_data_size - sizeof(daq_multi_cycle_header_t)) / DAQ_NUM_DATA_BLOCKS_PER_SECOND;
- cycle_data_size -= (cycle_data_size % 8);
- printf ("cycle data size = %d\t%d\n",cycle_data_size, max_data_size_mb);
- sleep(3);
- ifo_header->cycleDataSize = cycle_data_size;
- ifo_header->maxCycle = DAQ_NUM_DATA_BLOCKS_PER_SECOND;
-
-
- printf("nsys = %d\n",nsys);
- for(ii=0;ii<nsys;ii++) {
- printf("sys %d = %s\n",ii,sname[ii]);
- }
-
- // Make 0MQ socket connections
- for(ii=0;ii<nsys;ii++) {
- // Create a thread to receive data from each data server
- thread_index[ii].index = ii;
- thread_index[ii].src_iface = (niface ? local_iface[ii % niface]: 0);
- pthread_create(&thread_id[ii],NULL,rcvr_thread,(void *)&thread_index[ii]);
- }
-
- if(xmitData) {
- // Connect to Dolphin
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- // Set pointer to xmit header in Dolphin xmit data area.
- mywriteaddr = (char *)writeAddr;
- for(ii=0;ii<IX_BLOCK_COUNT;ii++) {
- xmitHeader[ii] = (daq_multi_cycle_header_t *)mywriteaddr;
- mywriteaddr += IX_BLOCK_SIZE;
- xmitDataOffset[ii] = IX_BLOCK_SIZE * ii + sizeof(struct daq_multi_cycle_header_t);
- }
- }
- //
-
- int nextCycle = 0;
- int64_t prev_sec_and_cycle = 0;
- start_acq = 1;
- int64_t mytime = 0;
- int64_t mylasttime = 0;
- int64_t myptime = 0;
- int64_t n_cycle_time = 0;
- int mytotaldcu = 0;
- char *zbuffer;
- size_t zbuffer_remaining = 0;
- int dc_datablock_size = 0;
- int datablock_size_running = 0;
- int datablock_size_mb_s = 0;
- static const int header_size = sizeof(daq_multi_dcu_header_t);
- char dcstatus[4096];
- char dcs[48];
- int edcuid[10];
- int estatus[10];
- int edbs[10];
- unsigned long ets = 0;
- int dataRdy[MAX_FE_COMPUTERS];
- int any_rdy = 0;
- int jj,kk;
- int sendLength = 0;
-
- int min_cycle_time = 1 << 30;
- int max_cycle_time = 0;
- int mean_cycle_time = 0;
- int pv_dcu_count = 0;
- int pv_total_datablock_size = 0;
- int endpoint_min_count = 1 << 30;
- int endpoint_max_count = 0;
- int endpoint_mean_count = 0;
- int cur_endpoint_ready_count;
- char endpoints_missed_buffer[256];
- int endpoints_missed_remaining;
- int missed_flag = 0;
- int missed_nsys[MAX_FE_COMPUTERS];
- //int64_t recv_time[MAX_FE_COMPUTERS];
- //int64_t min_recv_time = 0;
- int64_t cur_ref_time = 0;
- int recv_buckets[(MAX_DELAY_MS/5)+2];
- int entry_binned = 0;
- daq_multi_dcu_data_t* mx_cur_datablock = 0;
-
- mask_t dcu_received_mask;
- mask_t dcu_received_prev_mask;
- mask_t endpoint_received_mask;
- mask_t endpoint_received_prev_mask;
-
- int dcu_endpoint_lookup[DCU_COUNT];
- memset(dcu_endpoint_lookup, 0, sizeof(dcu_endpoint_lookup));
-
- mask_clear(&dcu_received_mask);
- mask_clear(&dcu_received_prev_mask);
- mask_clear(&endpoint_received_mask);
- mask_clear(&endpoint_received_prev_mask);
-
- SimplePV pvs[] = {
- {
- "RECV_BIN_MISS",
- SIMPLE_PV_INT,
- &recv_buckets[0],
- 1,
- -1,
- 1,
- -1,
- },
- {
- "RECV_BIN_0",
- SIMPLE_PV_INT,
- &recv_buckets[1],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_1",
- SIMPLE_PV_INT,
- &recv_buckets[2],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_2",
- SIMPLE_PV_INT,
- &recv_buckets[3],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_3",
- SIMPLE_PV_INT,
- &recv_buckets[4],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_4",
- SIMPLE_PV_INT,
- &recv_buckets[5],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_5",
- SIMPLE_PV_INT,
- &recv_buckets[6],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_6",
- SIMPLE_PV_INT,
- &recv_buckets[7],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_7",
- SIMPLE_PV_INT,
- &recv_buckets[8],
- nsys,
- -1,
- nsys,
- -1,
- },
- {
- "RECV_BIN_8",
- SIMPLE_PV_INT,
- &recv_buckets[9],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_MIN_MS",
- SIMPLE_PV_INT,
- &min_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MAX_MS",
- SIMPLE_PV_INT,
- &max_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MEAN_MS",
- SIMPLE_PV_INT,
- &mean_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "DCU_COUNT",
- SIMPLE_PV_INT,
- &pv_dcu_count,
-
- 120,
- 0,
- 115,
- 0,
- },
- {
- "DATA_SIZE",
- SIMPLE_PV_INT,
- &pv_total_datablock_size,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1*1024*1024,
- },
- {
- "ENDPOINT_MIN_COUNT",
- SIMPLE_PV_INT,
- &endpoint_min_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "ENDPOINT_MAX_COUNT",
- SIMPLE_PV_INT,
- &endpoint_max_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "ENDPOINT_MEAN_COUNT",
- SIMPLE_PV_INT,
- &endpoint_mean_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "DATA_BLOCK_SIZE",
- SIMPLE_PV_INT,
- &dc_datablock_size,
-
- DAQ_DCU_BLOCK_SIZE,
- 0,
- (DAQ_DCU_BLOCK_SIZE*9)/10,
- 1*1024*1024,
-
- },
- {
- "DATA_BLOCK_SIZE_MB_PER_S",
- SIMPLE_PV_INT,
- &datablock_size_mb_s,
-
- DAQ_TRANSIT_MAX_DC_BYTE_SEC/(1024*1024),
- 0,
- (DAQ_TRANSIT_MAX_DC_BYTE_SEC/(1024*1024))*9/10,
- 1,
- },
- {
- "ENDPOINTS_MISSED",
- SIMPLE_PV_STRING,
- endpoints_missed_buffer,
-
- 0, 0, 0, 0,
- },
-
- };
- if (pv_debug_pipe_name)
- {
- pv_debug_pipe = open(pv_debug_pipe_name, O_NONBLOCK | O_RDWR, 0);
- if (pv_debug_pipe < 0) {
- fprintf(stderr, "Unable to open %s for writting (pv status)\n", pv_debug_pipe_name);
- exit(1);
- }
- }
-
- missed_flag = 1;
- memset(&missed_nsys[0], 0, sizeof(missed_nsys));
- memset(recv_buckets, 0, sizeof(recv_buckets));
- do {
- // reset masks
- mask_copy(&dcu_received_prev_mask, &dcu_received_mask);
- mask_copy(&endpoint_received_prev_mask, &endpoint_received_mask);
- mask_clear(&dcu_received_mask);
- mask_clear(&endpoint_received_mask);
-
-// // Wait up to 100ms until received data from at least 1 FE or timeout
-// do {
-// usleep(2000);
-// for(ii=0;ii<nsys;ii++) {
-// if(nextCycle == thread_cycle[ii]) any_rdy = 1;
-// }
-// timeout += 1;
-// }while(!any_rdy && timeout < 50);
-//
-// // Wait up to delay_ms ms in 1/10ms intervals until data received from everyone or timeout
-// timeout = 0;
-// do {
-// usleep(100);
-// for(ii=0;ii<nsys;ii++) {
-// if(nextCycle == thread_cycle[ii] && !dataRdy[ii]) threads_rdy ++;
-// if(nextCycle == thread_cycle[ii]) dataRdy[ii] = 1;
-// }
-// timeout += 1;
-// }while(threads_rdy < nsys && timeout < delay_cycles);
-// if(timeout >= 100) rcv_errors += (nsys - threads_rdy);
-
- any_rdy = wait_for_data(nsys, dataRdy, &nextCycle, prev_sec_and_cycle, &rcv_errors);
-
- if(any_rdy) {
- // Timing diagnostics
- mytime = s_clock();
- myptime = mytime - mylasttime;
- mylasttime = mytime;
- if (myptime < min_cycle_time) {
- min_cycle_time = myptime;
- }
- if (myptime > max_cycle_time) {
- max_cycle_time = myptime;
- }
- mean_cycle_time += myptime;
- ++n_cycle_time;
- if(do_verbose)printf("Data rdy for cycle = %d\t\t%ld\n",nextCycle,myptime);
-
- // Reset total DCU counter
- mytotaldcu = 0;
- // Reset total DC data size counter
- dc_datablock_size = 0;
- // Get pointer to next data block in shared memory
- nextData = (char *)ifo_data;
- nextData += cycle_data_size * nextCycle;
- ifoDataBlock = (daq_multi_dcu_data_t *)nextData;
- zbuffer = (char *)nextData + header_size;
- zbuffer_remaining = cycle_data_size - header_size;
-
- cur_endpoint_ready_count = 0;
- endpoints_missed_remaining = sizeof(endpoints_missed_buffer);
- endpoints_missed_buffer[0] = '\0';
- //min_recv_time = 0x7fffffffffffffff;
- // Loop over all data buffers received from FE computers
- for(ii=0;ii<nsys;ii++) {
- //recv_time[ii] = -1;
- if(dataRdy[ii]) {
- mask_set_entry(&endpoint_received_mask, ii);
-
- //recv_time[ii] = dataRecvTime[ii];
- //if (recv_time[ii] < min_recv_time) {
- // min_recv_time = recv_time[ii];
- //}
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "+++%s\n", sname[ii]);
- }
- ++cur_endpoint_ready_count;
- mx_cur_datablock = &mxDataBlockSingle[ii][nextCycle];
- int myc = mx_cur_datablock->header.dcuTotalModels;
- // For each model, copy over data header information
- for(jj=0;jj<myc;jj++) {
- // Copy data header information
- ifoDataBlock->header.dcuheader[mytotaldcu].dcuId = mx_cur_datablock->header.dcuheader[jj].dcuId;
- ifoDataBlock->header.dcuheader[mytotaldcu].fileCrc = mx_cur_datablock->header.dcuheader[jj].fileCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].status = mx_cur_datablock->header.dcuheader[jj].status;
- ifoDataBlock->header.dcuheader[mytotaldcu].cycle = mx_cur_datablock->header.dcuheader[jj].cycle;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeSec = mx_cur_datablock->header.dcuheader[jj].timeSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeNSec = mx_cur_datablock->header.dcuheader[jj].timeNSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataCrc = mx_cur_datablock->header.dcuheader[jj].dataCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataBlockSize = mx_cur_datablock->header.dcuheader[jj].dataBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpBlockSize = mx_cur_datablock->header.dcuheader[jj].tpBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount = mx_cur_datablock->header.dcuheader[jj].tpCount;
-
- mask_set_entry(&dcu_received_mask, ifoDataBlock->header.dcuheader[mytotaldcu].dcuId);
- dcu_endpoint_lookup[ifoDataBlock->header.dcuheader[mytotaldcu].dcuId] = ii;
-
- for(kk=0;kk<DAQ_GDS_MAX_TP_NUM ;kk++)
- ifoDataBlock->header.dcuheader[mytotaldcu].tpNum[kk] = mx_cur_datablock->header.dcuheader[jj].tpNum[kk];
- edbs[mytotaldcu] = mx_cur_datablock->header.dcuheader[jj].tpBlockSize + mx_cur_datablock->header.dcuheader[jj].dataBlockSize;
- // Get some diags
- if(ifoDataBlock->header.dcuheader[mytotaldcu].status != 0xbad)
- ets = mx_cur_datablock->header.dcuheader[jj].timeSec;
- estatus[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].status;
- edcuid[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].dcuId;
- // Increment total DCU count
- mytotaldcu ++;
- }
- // Get the size of the data to transfer
- int mydbs = mx_cur_datablock->header.fullDataBlockSize;
- // Get pointer to data in receive data block
- char *mbuffer = (char *)&mx_cur_datablock->dataBlock[0];
- if (mydbs > zbuffer_remaining)
- {
- fprintf(stderr, "Buffer overflow found. Attempting to write %d bytes to zbuffer which has %d bytes remainging\n",
- (int)mydbs, (int)zbuffer_remaining);
- abort();
- }
- // Copy data from receive buffer to shared memory
- memcpy(zbuffer,mbuffer,mydbs);
- // Increment shared memory data buffer pointer for next data set
- zbuffer += mydbs;
- // Calc total size of data block for this cycle
- dc_datablock_size += mydbs;
- } else {
- if (recv_map_fp)
- {
- record_recv_map_for_cycles = 3;
- }
- missed_nsys[ii] |= missed_flag;
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "---%s\n", sname[ii]);
- }
- }
- }
- if (cur_endpoint_ready_count < endpoint_min_count) {
- endpoint_min_count = cur_endpoint_ready_count;
- }
- if (cur_endpoint_ready_count > endpoint_max_count) {
- endpoint_max_count = cur_endpoint_ready_count;
- }
- endpoint_mean_count += cur_endpoint_ready_count;
- // Write total data block size to shared memory header
- ifoDataBlock->header.fullDataBlockSize = dc_datablock_size;
- // Write total dcu count to shared memory header
- ifoDataBlock->header.dcuTotalModels = mytotaldcu;
- // Set multi_cycle head cycle to indicate data ready for this cycle
- ifo_header->curCycle = nextCycle;
- xmitBlockNum = nextCycle % IX_BLOCK_COUNT;
-
- // Calc IX message size
- sendLength = header_size + ifoDataBlock->header.fullDataBlockSize;
- if (nextCycle == 0) {
- memset(recv_buckets, 0, sizeof(recv_buckets));
- }
- /*for (ii = 0; ii < nsys; ++ii) {
- cur_ref_time = 0;
- recv_time[ii] -= min_recv_time;
- entry_binned = 0;
- for (jj = 0; jj < sizeof(recv_buckets)/sizeof(recv_buckets[0]); ++jj) {
- if (recv_time[ii] < cur_ref_time) {
- ++recv_buckets[jj];
- entry_binned = 1;
- break;
- }
- cur_ref_time += 5;
- }
- if (!entry_binned) {
- ++recv_buckets[sizeof(recv_buckets)/sizeof(recv_buckets[0])-1];
- }
- }*/
- datablock_size_running += dc_datablock_size;
- if (nextCycle == 0) {
- datablock_size_mb_s = datablock_size_running / (1024*1024);
- pv_dcu_count = mytotaldcu;
- pv_total_datablock_size = dc_datablock_size;
- mean_cycle_time = (n_cycle_time > 0 ? mean_cycle_time / n_cycle_time : 1 << 31);
- endpoint_mean_count = (n_cycle_time > 0 ? endpoint_mean_count/n_cycle_time : 1<<31);
-
- endpoints_missed_remaining = sizeof(endpoints_missed_buffer)-1;
- endpoints_missed_buffer[0] = '\0';
- for (ii = 0; ii < sizeof(missed_nsys)/sizeof(missed_nsys[0]) && endpoints_missed_remaining > 0; ++ii) {
- if (missed_nsys[ii]) {
- char tmp[80];
- int count = snprintf(tmp, sizeof(tmp), "%s(%x) ", sname[ii], missed_nsys[ii]);
- if (count < sizeof(tmp)) {
- strncat(endpoints_missed_buffer, tmp, count);
- endpoints_missed_remaining -= count;
- }
- }
- missed_nsys[ii] = 0;
- }
- send_pv_update(pv_debug_pipe, pv_prefix, pvs, sizeof(pvs)/sizeof(pvs[0]));
-
- if (do_verbose) {
- printf("\nData rdy for cycle = %d\t\tTime Interval = %ld msec\n", nextCycle, myptime);
- printf("Min/Max/Mean cylce time %ld/%ld/%ld msec over %ld cycles\n", min_cycle_time, max_cycle_time,
- mean_cycle_time, n_cycle_time);
- printf("Total DCU = %d\t\t\tBlockSize = %d\n", mytotaldcu, dc_datablock_size);
- print_diags(mytotaldcu, nextCycle, sendLength, ifoDataBlock, edbs);
- }
- n_cycle_time = 0;
- min_cycle_time = 1 << 30;
- max_cycle_time = 0;
- mean_cycle_time = 0;
-
- endpoint_min_count = nsys;
- endpoint_max_count = 0;
- endpoint_mean_count = 0;
-
- missed_flag = 1;
- datablock_size_running = 0;
- } else {
- missed_flag <<= 1;
- }
- if(xmitData) {
- if (sendLength > IX_BLOCK_SIZE)
- {
- fprintf(stderr, "Buffer overflow. Sending %d bytes into a dolphin block that holds %d\n",
- (int)sendLength, (int)IX_BLOCK_SIZE);
- abort();
- }
- // WRITEDATA to Dolphin Network
- SCIMemCpy(sequence,nextData, remoteMap,xmitDataOffset[xmitBlockNum],sendLength,memcpyFlag,&error);
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- fprintf(stderr,"For reference the expected error codes are:\n");
- fprintf(stderr,"SCI_ERR_OUT_OF_RANGE = 0x%x\n", SCI_ERR_OUT_OF_RANGE);
- fprintf(stderr,"SCI_ERR_SIZE_ALIGNMENT = 0x%x\n", SCI_ERR_SIZE_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_OFFSET_ALIGNMENT = 0x%x\n", SCI_ERR_OFFSET_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_TRANSFER_FAILED = 0x%x\n", SCI_ERR_TRANSFER_FAILED);
- return error;
- }
- // Set data header information
- xmitHeader[xmitBlockNum]->maxCycle = ifo_header->maxCycle;
- // xmitHeader->cycleDataSize = ifo_header->cycleDataSize;
- xmitHeader[xmitBlockNum]->cycleDataSize = sendLength;;
- // xmitHeader->msgcrc = myCrc;
- // Send cycle last as indication of data ready for receivers
- xmitHeader[xmitBlockNum]->curCycle = ifo_header->curCycle;
- // Have to flush the buffers to make data go onto Dolphin network
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-
- if (recv_map_fp && record_recv_map_for_cycles > 0)
- {
- dump_recv_mask(recv_map_fp, nsys, ifoDataBlock->header.dcuheader[0].timeSec, nextCycle, &endpoint_received_mask);
- --record_recv_map_for_cycles;
- }
- }
- sprintf(dcstatus,"%ld ",ets);
- for(ii=0;ii<mytotaldcu;ii++) {
- sprintf(dcs,"%d %d %d ",edcuid[ii],estatus[ii],edbs[ii]);
- strcat(dcstatus,dcs);
- }
-
- dump_dcu_mask_diff(&dcu_received_prev_mask, &dcu_received_mask, dcu_endpoint_lookup);
- dump_endpoint_mask_diff(&endpoint_received_prev_mask, &endpoint_received_mask);
- // Increment cycle count
- nextCycle ++;
- nextCycle %= 16;
- }while (keepRunning); // End of infinite loop
-
- // Stop Rcv Threads
- printf("stopping threads %d \n",nsys);
- stop_working_threads = 1;
-
- // Wait for threads to stop
- sleep(2);
- if(xmitData) {
- printf("closing out ix\n");
- // Cleanup the Dolphin connections
- error = dolphin_closeout();
- }
- printf("closing out zmq\n");
- // Cleanup the ZMQ connections
- for(ii=0;ii<nsys;ii++) {
- if (daq_subscriber[ii]) zmq_close(daq_subscriber[ii]);
- if (daq_context[ii]) zmq_ctx_destroy(daq_context[ii]);
- }
-
- exit(0);
-}
diff --git a/src/ix_stream/zmq_rcv_ix_xmit.cc b/src/ix_stream/zmq_rcv_ix_xmit.cc
deleted file mode 100644
index 12de64d46..000000000
--- a/src/ix_stream/zmq_rcv_ix_xmit.cc
+++ /dev/null
@@ -1,1471 +0,0 @@
-//
-/// @file zmq_rcv_ix_xmitcc
-/// @brief DAQ data concentrator code. Receives data via ZMQ and sends via Dolphin IX..
-//
-
-#include <iostream>
-
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/file.h>
-#include <stdio.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include "../drv/crc.c"
-#include <zmq.h>
-#include <assert.h>
-#include <time.h>
-#include "../include/daqmap.h"
-#include "../include/daq_core.h"
-#include "dc_utils.h"
-
-#include "zmq_transport.h"
-#include "simple_pv.h"
-
-#include <stdarg.h>
-
-#define __CDECL
-
-#define DO_HANDSHAKE 0
-
-
-#include "dolphin_common.hh"
-
-
-template <typename T>
-class lock_guard;
-
-/***
- * A simple lock guard specialized for pthread spinlocks.
- * Use this to ensure locks are always released.
- *
- * @note Creation of the object is lock acquisition
- * Destruction of the object is lock release
- */
-template<>
-class lock_guard<pthread_spinlock_t>
-{
-public:
- lock_guard(pthread_spinlock_t& sp): sp_(sp)
- {
- pthread_spin_lock(&sp_);
- }
- ~lock_guard()
- {
- pthread_spin_unlock(&sp_);
- }
-private:
- lock_guard(const lock_guard& other);
- lock_guard& operator=(const lock_guard& other);
-
- pthread_spinlock_t& sp_;
-};
-
-#define MIN_DELAY_MS 5
-#define MAX_DELAY_MS 40
-
-#define MAX_FE_COMPUTERS 64
-
-#define MAX_RECEIVE_BUFFERS 4
-
-extern "C" {
-extern volatile void *findSharedMemorySize(const char *, int);
-}
-
-int do_verbose = 0;
-int debug_zmq = 0;
-bool debug_wait_timings = false;
-
-struct thread_info {
- int index;
- char *src_iface;
-};
-struct thread_mon_info {
- int index;
- void *ctx;
-};
-
-typedef struct mask_t {
- int data[DCU_COUNT];
-} mask_t;
-
-typedef struct receive_map_entry_t {
- int64_t gps_sec_and_cycle;
- int64_t s_clock;
-} receive_map_entry_t;
-
-/// \brief structure to hold when data has been received
-typedef struct receive_map_t
-{
- receive_map_entry_t data[MAX_RECEIVE_BUFFERS][MAX_FE_COMPUTERS]; /// indexed by cycle then machine
-} receive_map_t;
-
-pthread_spinlock_t receive_map_lock;
-receive_map_t receive_map;
-
-/// \brief given a gps second + a cycle count merge them into one value
-static inline int64_t calc_gps_sec_and_cycle(int64_t gps_sec, int64_t cycle)
-{
- return (gps_sec << 4) | (0x0f & cycle);
-}
-
-// helper to extract seconds counts out of a combined gps sec + cycle value
-static inline int64_t extract_gps(int64_t gps_sec_and_cycle)
-{
- return gps_sec_and_cycle >> 4;
-}
-
-// helper to extract cycle counts out of a combined gps sec + cycle value
-static inline int64_t extract_cycle(int64_t gps_sec_and_cycle)
-{
- return gps_sec_and_cycle & 0x0f;
-}
-
-
-static inline void mask_clear(mask_t* mask)
-{
- memset(mask->data, 0, sizeof(mask->data));
-}
-
-static inline void mask_set_entry(mask_t* mask, int index)
-{
- const unsigned int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- if (((unsigned int)index) < MAX_INDEX)
- {
- mask->data[index] = 1;
- }
-}
-
-static inline int mask_get_entry(mask_t* mask, int index)
-{
- const unsigned int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- return (((unsigned int)index) < MAX_INDEX ? mask->data[index] : 0);
-}
-
-static inline int mask_diffs(mask_t* input1, mask_t* input2, mask_t* dest)
-{
- const int MAX_INDEX = sizeof(input1->data)/sizeof(input1->data[0]);
- int diffs = 0;
- int i = 0;
- for(; i < MAX_INDEX; ++i)
- {
- diffs += (dest->data[i] = (input1->data[i] != input2->data[i]));
- }
- return diffs;
-}
-
-static inline int mask_differs(mask_t* input1, mask_t* input2)
-{
- const int MAX_INDEX = sizeof(input1->data)/sizeof(input1->data[0]);
- int i = 0;
- for (; i < MAX_INDEX; ++i)
- {
- if (input1->data[i] != input2->data[i])
- {
- return 1;
- }
- }
- return 0;
-}
-
-static inline void mask_dump(mask_t* mask, const char* description)
-{
- printf("%s\n", description);
- const int MAX_INDEX = sizeof(mask->data)/sizeof(mask->data[0]);
- int i = 0;
- for (; i < MAX_INDEX; ++i)
- {
- if (mask->data[i])
- {
- printf("(%d) ", i);
- }
- }
- printf("\n");
-}
-
-static inline void mask_copy(mask_t* dest, const mask_t* src)
-{
- memcpy(dest, src, sizeof(*dest));
-}
-
-static void dump_dcu_mask_diff(mask_t* prev, mask_t* cur, int dcu_endpoint_lookup[DCU_COUNT])
-{
- const int MAX_INDEX = sizeof(cur->data)/sizeof(cur->data[0]);
- if (!mask_differs(prev, cur))
- {
- return;
- }
- printf("\nDCU state change:\n");
- int i = 0;
- for (i = 0; i < MAX_INDEX; ++i)
- {
- if (prev->data[i] < cur->data[i])
- {
- printf("(+%d)e%d ", i, dcu_endpoint_lookup[i]);
- }
- else if (prev->data[i] > cur->data[i])
- {
- printf("(-%d)e%d ", i, dcu_endpoint_lookup[i]);
- }
- }
-}
-
-static void dump_endpoint_mask_diff(mask_t* prev, mask_t* cur)
-{
- const int MAX_INDEX = sizeof(cur->data)/sizeof(cur->data[0]);
- if (!mask_differs(prev, cur))
- {
- return;
- }
- printf("\nEndpoint state change:\n");
- int i = 0;
- for (i = 0; i < MAX_INDEX; ++i)
- {
- if (prev->data[i] < cur->data[i])
- {
- printf("(+%d) ", i);
- }
- else if (prev->data[i] > cur->data[i])
- {
- printf("(-%d) ", i);
- }
- }
- printf("\n");
-}
-
-static void receive_map_init()
-{
- memset(&receive_map, 0, sizeof(receive_map));
- pthread_spin_init(&receive_map_lock, PTHREAD_PROCESS_PRIVATE);
-}
-
-static void receive_map_mark(int fe_index, long gps_sec, int64_t cycle, int64_t clock_val)
-{
- int64_t gps_sec_and_cycle = 0;
-
- int cycle_index = (int)(cycle % MAX_RECEIVE_BUFFERS);
- gps_sec_and_cycle = calc_gps_sec_and_cycle(gps_sec, cycle);
-
- lock_guard<pthread_spinlock_t> lock(receive_map_lock);
- receive_map.data[cycle_index][fe_index].gps_sec_and_cycle = gps_sec_and_cycle;
- receive_map.data[cycle_index][fe_index].s_clock = clock_val;
-}
-
-static void receive_map_copy_to_local(receive_map_t *dest)
-{
- if (!dest)
- {
- return;
- }
- lock_guard<pthread_spinlock_t> lock_(receive_map_lock);
- memcpy(dest, &receive_map, sizeof(receive_map));
-}
-
-static void dump_recv_mask(FILE *f, int nsys, int64_t gps_sec, int block_index, mask_t* received_mask)
-{
- int fe_index = 0, cycle_index = 0;
- receive_map_t local_map;
- if (!f)
- {
- return;
- }
- receive_map_copy_to_local(&local_map);
- fprintf(f, "----------------------------------------------\n");
- fprintf(f, "-- Gps %ld:%d\n", (long int)gps_sec, (int)block_index);
- fprintf(f, "----------------------------------------------\n");
-
- block_index %= MAX_RECEIVE_BUFFERS;
-
- for (cycle_index = MAX_RECEIVE_BUFFERS - 1; cycle_index >= 0 ; --cycle_index)
- {
- fprintf(f, "% 2d) ", cycle_index);
- for (fe_index = 0; fe_index < nsys; ++fe_index)
- {
- fprintf(f, "|%s%ld:%d (%ld) ",
- (cycle_index != block_index ? " " :
- (mask_get_entry(received_mask, fe_index) ? " " : "*")),
- (long int) local_map.data[cycle_index][fe_index].gps_sec_and_cycle >> 4,
- 0x0f & (int) local_map.data[cycle_index][fe_index].gps_sec_and_cycle,
- (long int) local_map.data[cycle_index][fe_index].s_clock);
- }
- fprintf(f, "|\n");
- }
-
- fprintf(f, "----------------------------------------------\n\n");
-}
-
-static void dump_local_recv_mask(receive_map_t& local_map, int nsys, int64_t expected_sec, int cycle)
-{
- using std::cout;
-
- cout << "----------------------------------------------\n";
- cout << "-- cycle = " << cycle << " expected sec = "
- << extract_gps(expected_sec) << ":" << extract_cycle(expected_sec) << "\n";
- cout << "----------------------------------------------\n";
-
- int cycle_index = cycle % MAX_RECEIVE_BUFFERS;
-
- for (cycle_index = MAX_RECEIVE_BUFFERS - 1; cycle_index >= 0; --cycle_index)
- {
- cout << cycle_index << ") ";
- for (int fe_index = 0; fe_index < nsys; ++fe_index)
- {
- cout << "| " << extract_gps(local_map.data[cycle_index][fe_index].gps_sec_and_cycle)
- << ":" << extract_cycle(local_map.data[cycle_index][fe_index].gps_sec_and_cycle)
- << " (" << local_map.data[cycle_index][fe_index].s_clock << ") ";
- }
- cout << "|\n";
- }
-}
-
-
-
-struct thread_info thread_index[DCU_COUNT];
-void *daq_context[DCU_COUNT];
-void *daq_subscriber[DCU_COUNT];
-char *sname[DCU_COUNT]; // Names of FE computers serving DAQ data
-char *local_iface[MAX_FE_COMPUTERS];
-daq_multi_dcu_data_t mxDataBlockSingle[MAX_FE_COMPUTERS][MAX_RECEIVE_BUFFERS];
-pthread_spinlock_t mxDataBlockSingle_lock[MAX_FE_COMPUTERS][MAX_RECEIVE_BUFFERS];
-const int mc_header_size = sizeof(daq_multi_cycle_header_t);
-int stop_working_threads = 0;
-int start_acq = 0;
-static volatile int keepRunning = 1;
-//int thread_cycle[MAX_FE_COMPUTERS];
-//int thread_timestamp[MAX_FE_COMPUTERS];
-int rcv_errors = 0;
-
-void
-usage()
-{
- fprintf(stderr, "Usage: zmq_rcv_ix_xmit [args] -s server names -m shared memory size -g IX channel \n");
- fprintf(stderr, "-l filename - log file name\n");
- fprintf(stderr, "-b buffername - name of the mbuf to write to\n");
- fprintf(stderr, "-s - server names separated by spaces eg \"x1lsc0 x1susex etc.\"\n");
- fprintf(stderr, "-v - verbose prints diag test data\n");
- fprintf(stderr, "-g - Dolphin IX channel to xmit on\n");
- fprintf(stderr, "-p - Debug pv prefix, requires -P as well\n");
- fprintf(stderr, "-P - Path to a named pipe to send PV debug information to\n");
- fprintf(stderr, "-d - Max delay in milli seconds to wait for a FE to send data, defaults to 10\n");
- fprintf(stderr, "-L ifaces - local interfaces to listen on [used for load balancing], eg \"eth0 eth1\"\n");
- fprintf(stderr, "-z - output zmq debug information (state changes)\n");
- fprintf(stderr, "-X path - output history of received data to given path\n");
- fprintf(stderr, "-h - help\n");
-}
-
-// *************************************************************************
-// Timing Diagnostic Routine
-// *************************************************************************
-static int64_t
-s_clock (void)
-{
-struct timeval tv;
- gettimeofday (&tv, NULL);
- return (int64_t) (tv.tv_sec * 1000 + tv.tv_usec / 1000);
-}
-
-// *************************************************************************
-// Catch Control C to end cod in controlled manner
-// *************************************************************************
-void intHandler(int dummy) {
- keepRunning = 0;
-}
-
-void sigpipeHandler(int dummy)
-{}
-
-// **********************************************************************************************
-void print_diags(int nsys, int lastCycle, int sendLength, daq_multi_dcu_data_t *ixDataBlock,int dbs[]) {
-// **********************************************************************************************
- int ii = 0;
- // Print diags in verbose mode
- printf("Receive errors = %d\n",rcv_errors);
- printf("Time = %d\t size = %d\n",ixDataBlock->header.dcuheader[0].timeSec,sendLength);
- printf("DCU ID\tCycle \t TimeSec\tTimeNSec\tDataSize\tTPCount\tTPSize\tXmitSize\n");
- for(ii=0;ii<nsys;ii++) {
- printf("%d",ixDataBlock->header.dcuheader[ii].dcuId);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].cycle);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].timeSec);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].timeNSec);
- printf("\t\t%d",ixDataBlock->header.dcuheader[ii].dataBlockSize);
- printf("\t\t%d",ixDataBlock->header.dcuheader[ii].tpCount);
- printf("\t%d",ixDataBlock->header.dcuheader[ii].tpBlockSize);
- printf("\t%d",dbs[ii]);
- printf("\n ");
- }
-}
-
-const char *pevent_zmq(int event, char *buffer, size_t max_size)
-{
- if (!buffer || max_size < 1) {
- return "";
- }
- switch (event) {
- default:
- snprintf(buffer, max_size, "%d", event);
- break;
- case ZMQ_EVENT_CONNECTED:
- strncpy(buffer, "CONNECTED", max_size);
- break;
- case ZMQ_EVENT_CONNECT_DELAYED:
- strncpy(buffer, "CONNECT_DELAYED", max_size);
- break;
- case ZMQ_EVENT_CONNECT_RETRIED:
- strncpy(buffer, "CONNECT_RETRIED", max_size);
- break;
- case ZMQ_EVENT_LISTENING:
- strncpy(buffer, "LISTENING", max_size);
- break;
- case ZMQ_EVENT_BIND_FAILED:
- strncpy(buffer, "BIND_FAILED", max_size);
- break;
- case ZMQ_EVENT_ACCEPTED:
- strncpy(buffer, "ACCEPTED", max_size);
- break;
- case ZMQ_EVENT_ACCEPT_FAILED:
- strncpy(buffer, "ACCEPT_FAILED", max_size);
- break;
- case ZMQ_EVENT_CLOSED:
- strncpy(buffer, "CLOSED", max_size);
- break;
- case ZMQ_EVENT_CLOSE_FAILED:
- strncpy(buffer, "CLOSE_FAILED", max_size);
- break;
- case ZMQ_EVENT_DISCONNECTED:
- strncpy(buffer, "DISCONNECTED", max_size);
- break;
- case ZMQ_EVENT_MONITOR_STOPPED:
- strncpy(buffer, "MONITOR_STOPPED", max_size);
- break;
- }
- buffer[max_size-1] = '\0';
- return buffer;
-}
-
-// *****
-// monitoring thread callback
-// *****
-void *rcvr_thread_mon(void *args)
-{
- int rc;
- char msg_buf[100];
- struct thread_mon_info *info = (struct thread_mon_info*)args;
-
- void *s = zmq_socket(info->ctx, ZMQ_PAIR);
- rc = zmq_connect(s, "inproc://monitor.req");
- while (1)
- {
- zmq_msg_t msg;
- zmq_msg_init(&msg);
- rc = zmq_recvmsg(s, &msg, 0);
- if (rc == -1) break;
- unsigned short *event = (unsigned short*)zmq_msg_data(&msg);
- unsigned int *value = (unsigned int*)(event+1);
- fprintf(stderr, "%s) %d - %s\n", pevent_zmq(*event, msg_buf, sizeof(msg_buf)), *value, sname[info->index]);
- if (zmq_msg_more(&msg)) {
- zmq_msg_init(&msg);
- zmq_msg_recv(&msg, s, 0);
- }
- }
- zmq_close(s);
- delete info;
- return NULL;
-}
-
-// *************************************************************************
-// Thread for receiving DAQ data via ZMQ
-// *************************************************************************
-void *rcvr_thread(void *arg) {
- struct thread_mon_info* mon_info = 0;
- struct thread_info* my_info = (struct thread_info*)arg;
- int mt = my_info->index;
- printf("myarg = %d on iface %s\n", mt, (my_info->src_iface ? my_info->src_iface : "default interface"));
- int cycle = 0;
- daq_multi_dcu_data_t *mxDataBlock;
- char loc[256];
-
- void *zctx = 0;
- void *zsocket = 0;
- int rc = 0;
- pthread_t mon_th_id;
- int buffer_index = 0;
-
-
- zctx = zmq_ctx_new();
- zsocket = zmq_socket(zctx, ZMQ_SUB);
-
- if (debug_zmq) {
- zmq_socket_monitor(zsocket, "inproc://monitor.req", ZMQ_EVENT_ALL);
- mon_info = new thread_mon_info; // calloc(1, sizeof(struct thread_mon_info));
- memset((void*)mon_info, 0, sizeof(*mon_info));
- if (!mon_info) {
- fprintf(stderr, "Unable to initialize monitoring thread for %s\n", sname[mt]);
- } else {
- mon_info->index = mt;
- mon_info->ctx = zctx;
- pthread_create(&mon_th_id, NULL, rcvr_thread_mon, mon_info);
- }
- }
-
- rc = zmq_setsockopt(zsocket, ZMQ_SUBSCRIBE, "", 0);
- assert(rc == 0);
- if (!dc_generate_connection_string(loc, sname[mt], sizeof(loc), my_info->src_iface)) {
- fprintf(stderr, "Unable to create connection string for '%s'\n", sname[mt]);
- exit(1);
- }
- dc_set_zmq_options(zsocket);
- rc = zmq_connect(zsocket, loc);
- assert(rc == 0);
-
-
- printf("Starting receive loop for thread %d %s\n", mt, loc);
- char *daqbuffer = (char *)&mxDataBlockSingle[mt];
- mxDataBlock = (daq_multi_dcu_data_t *)daqbuffer;
- do {
- unsigned int gps_sec = 0;
- buffer_index = zmq_recv_daq_multi_dcu_t_into_buffer((daq_multi_dcu_data_t*)daqbuffer, mxDataBlockSingle_lock[mt], zsocket, MAX_RECEIVE_BUFFERS, &gps_sec, &cycle);
-
- // Get the message DAQ cycle number
- //cycle = mxDataBlock[buffer_index].header.dcuheader[0].cycle;
- // Pass cycle and timestamp data back to main process
- int64_t cur_time = s_clock();
- //receive_map_mark(mt, mxDataBlock[buffer_index].header.dcuheader[0].timeSec, cycle, cur_time);
- receive_map_mark(mt, gps_sec, cycle, cur_time);
- //printf("#### thread %d wrote %ld:%d at %ld ####\n", (int)mt, (long int)mxDataBlock[buffer_index].header.dcuheader[0].timeSec, (int)cycle, (long int)cur_time);
- // Run until told to stop by main thread
- } while(!stop_working_threads);
- printf("Stopping thread %d\n",mt);
- usleep(200000);
-
- zmq_close(zsocket);
- zmq_ctx_destroy(zctx);
- return(0);
-
-}
-
-int
-wait_for_data(int nsys, int dataReady[MAX_FE_COMPUTERS], int *nextCycle, int64_t *prev_sec_and_cycle, int *rcv_errors, int *report_extra_cycles)
-{
- const int max_extra_cycles = 60;
- const int base_deadline_value_ms = 60;
- static int extra_cycles = max_extra_cycles;
- int ii = 0;
- int wait_count = 0;
- int ready_count = 0;
- int timeout = 0;
- int64_t deadline_ms = 0;
- int64_t remaining_time_ms = 1; // this must start non-zero
- int next_cycle = *nextCycle;
- int64_t expected_sec_and_cycle = *prev_sec_and_cycle; // calc_gps_sec_and_cycle(extract_gps(*prev_sec_and_cycle + 1), next_cycle);
- receive_map_t data_ready_map;
-
- using namespace std;
-
- if (*prev_sec_and_cycle == 0)
- {
- expected_sec_and_cycle = 0;
- }
- if (debug_wait_timings)
- {
- cout << "\n----- wait_for_data nsys="<<nsys<<" nextCycle="<<*nextCycle<<" prev_sec="<<*prev_sec_and_cycle<<"\n";
- cout << "expected_sec_and_cycle = " << expected_sec_and_cycle << "\n";
- }
-
- for (ii = 0; ii < nsys; ++ii)
- {
- dataReady[ii] = 0;
- }
-
- do {
- receive_map_copy_to_local(&data_ready_map);
-
- int cycle_index = next_cycle % MAX_RECEIVE_BUFFERS;
-
- for (ii = 0; ii < nsys; ++ii) {
- if (dataReady[ii]) {
- continue;
- }
- // FIXME: what to do if it is a different second?
- // For the first time through expected_sec_and_cycle == 0, so this grabs
- // the first entry.
- // Otherwise we handle the case of ignoring old data (ie a front end dropped out)
- // but what about something that is running fast, or where one FE is signifigantly
- // off on its timing?
- if (data_ready_map.data[cycle_index][ii].gps_sec_and_cycle >= expected_sec_and_cycle
- && data_ready_map.data[cycle_index][ii].gps_sec_and_cycle != 0
- && extract_cycle(data_ready_map.data[cycle_index][ii].gps_sec_and_cycle) == next_cycle)
- {
- dataReady[ii] = 1;
- ++ready_count;
- if (debug_wait_timings) {
- cout << "receivied system " << ii << " at " << data_ready_map.data[cycle_index][ii].s_clock \
- << " ready_count=" << ready_count << " remaining=" << remaining_time_ms << "\n";
- }
- if (deadline_ms == 0)
- {
- //cout << "waited " << wait_count << " cycles\n";
- wait_count = 0;
- *prev_sec_and_cycle = data_ready_map.data[cycle_index][ii].gps_sec_and_cycle + 1;
- int64_t current_time = s_clock();
- deadline_ms = data_ready_map.data[cycle_index][ii].s_clock + base_deadline_value_ms + extra_cycles;
- extra_cycles = 0;
- remaining_time_ms = deadline_ms - s_clock();
- //cout << "deadline set to " << deadline_ms << " cur_time = " << current_time << " remaining = " << remaining_time_ms << "\n";
- }
- int dcus[10];
- int dcu_count = 0;
- {
- lock_guard<pthread_spinlock_t> lock_(mxDataBlockSingle_lock[ii][cycle_index]);
- dcu_count = mxDataBlockSingle[ii][cycle_index].header.dcuTotalModels;
- for (int j = 0; j < dcu_count && j < sizeof(dcus)/sizeof(dcus[0]); ++j)
- {
- dcus[j] = mxDataBlockSingle[ii][cycle_index].header.dcuheader[j].dcuId;
- }
- }
- if (dcu_count == 0)
- {
- cerr << "!!!!!!!!! 0 DCUs found in a transmitted block \n";
- }
- if (debug_wait_timings) {
- cout << "Found " << dcu_count << " DCUs";
- for (int j = 0; j < dcu_count && j < sizeof(dcus) / sizeof(dcus[0]); ++j) {
- cout << " " << dcus[j];
- }
- cout << "\n";
- }
- }
- }
- if (ready_count != nsys) {
- if (deadline_ms == 0) {
- usleep(1000); /* 1ms */
- if (debug_wait_timings) {
- cout << "<sleep 1ms>";
- ++wait_count;
- if (wait_count > 0 && wait_count % 200 == 0) {
- std::cout << "<--- dumping receive map info for overlong wait -->\n";
- dump_local_recv_mask(data_ready_map, nsys, expected_sec_and_cycle, next_cycle);
- }
- }
- } else {
- int64_t cur_time = s_clock();
- remaining_time_ms = deadline_ms - cur_time;
- if (remaining_time_ms > 0) {
- //cout << "<sleep 0.2ms - cur_time = " << cur_time << " remaining = " << remaining_time_ms << "ms>";
- ++wait_count;
- usleep(200); /* 0.2ms */
- }
- }
- }
- } while (remaining_time_ms > 0 && ready_count != nsys);
- if (remaining_time_ms > 0)
- {
- extra_cycles += remaining_time_ms;
- if (extra_cycles > max_extra_cycles)
- {
- extra_cycles = max_extra_cycles;
- }
- }
- if (debug_wait_timings) {
- cout << "exiting wait loop, ready_count = " << ready_count << "\nremaining = " << remaining_time_ms
- << " waited " << wait_count << "cycles\n---------------\n";
- dump_local_recv_mask(data_ready_map, nsys, expected_sec_and_cycle, next_cycle);
- cout << "\n\n";
- cout.flush();
- }
- if (report_extra_cycles)
- {
- *report_extra_cycles = extra_cycles;
- }
- return ready_count;
-}
-
-
-// *************************************************************************
-// Main Process
-// *************************************************************************
-int
-main(int argc, char **argv)
-{
- pthread_t thread_id[MAX_FE_COMPUTERS];
- unsigned int nsys = 1; // The number of mapped shared memories (number of data sources)
- char *sysname;
- const char *buffer_name = "ifo";
- int c;
- int ii, jj, kk; // Loop counter
- int delay_ms = 10;
- int delay_cycles = 0;
- unsigned int niface = 0; // The number of local interfaces to split receives across.
- char *local_iface_names = 0;
-
- extern char *optarg; // Needed to get arguments to program
-
- // PV/debug information
- char *pv_prefix = 0;
- char *pv_debug_pipe_name = 0;
- int pv_debug_pipe = -1;
-
- // Declare shared memory data variables
- daq_multi_cycle_header_t *ifo_header;
- char *ifo;
- char *ifo_data;
- int cycle_data_size;
- daq_multi_dcu_data_t *ifoDataBlock;
- char *nextData;
- int max_data_size_mb = 100;
- int max_data_size = 0;
- char *mywriteaddr;
- daq_multi_cycle_header_t *xmitHeader[IX_BLOCK_COUNT];
- static int xmitDataOffset[IX_BLOCK_COUNT];
- int xmitBlockNum = 0;
-
- /* set up defaults */
- sysname = NULL;
- int xmitData = 0;
-
- receive_map_init();
- for (ii = 0; ii < MAX_FE_COMPUTERS; ++ii)
- {
- for (jj = 0; jj < MAX_RECEIVE_BUFFERS; ++jj)
- {
- pthread_spin_init(&mxDataBlockSingle_lock[ii][jj], PTHREAD_PROCESS_PRIVATE);
- }
- }
-
- FILE* receive_map_dump_file = 0;
- int receive_map_dump_counter = 0;
-
- // Get arguments sent to process
- while ((c = getopt(argc, argv, "b:hs:m:g:vp:P:d:DL:zX:")) != EOF) switch(c) {
- case 's':
- sysname = optarg;
- break;
- case 'v':
- do_verbose = 1;
- break;
- case 'z':
- debug_zmq = 1;
- break;
- case 'm':
- max_data_size_mb = atoi(optarg);
- if (max_data_size_mb < 20){
- printf("Min data block size is 20 MB\n");
- return -1;
- }
- if (max_data_size_mb > 100){
- printf("Max data block size is 100 MB\n");
- return -1;
- }
- break;
- case 'g':
- segmentId = atoi(optarg);
- xmitData = 1;
- break;
- case 'b':
- buffer_name = optarg;
- break;
- case 'p':
- pv_prefix = optarg;
- break;
- case 'P':
- pv_debug_pipe_name = optarg;
- break;
- case 'd':
- delay_ms = atoi(optarg);
- if (delay_ms < MIN_DELAY_MS || delay_ms > MAX_DELAY_MS) {
- printf("The delay factor must be between 5ms and 40ms\n");
- return -1;
- }
- break;
- case 'h':
- case 'L':
- local_iface_names = optarg;
- break;
- case 'D':
- debug_wait_timings = true;
- break;
- case 'X':
- receive_map_dump_file = fopen(optarg, "wt");
- break;
- default:
- usage();
- exit(1);
- }
- max_data_size = max_data_size_mb * 1024*1024;
- delay_cycles = delay_ms * 10;
-
- if (sysname == NULL) { usage(); exit(1); }
-
- // set up to catch Control C
- signal(SIGINT,intHandler);
- // setup to ignore sig pipe
- signal(SIGPIPE, sigpipeHandler);
-
- printf("Server name: %s\n", sysname);
-
- // Parse names of data servers
- sname[0] = strtok(sysname, " ");
- for(;;) {
- printf("%s\n", sname[nsys - 1]);
- char *s = strtok(0, " ");
- if (!s) break;
- // do not overflow our fixed size buffers
- assert(nsys < DCU_COUNT);
- sname[nsys] = s;
- nsys++;
- }
-
- // Parse local interface names
- if (local_iface_names) {
- local_iface[0] = strtok(local_iface_names, " ");
- niface++;
- for (;;) {
- printf("local interface %s\n", local_iface[niface-1]);
- char *s = strtok(0, " ");
- if (!s) break;
- assert(niface < MAX_FE_COMPUTERS);
- local_iface[niface] = s;
- niface++;
- }
- }
-
- // Get pointers to local DAQ mbuf
- ifo = reinterpret_cast<char *>(const_cast<void*>(findSharedMemorySize(buffer_name,max_data_size_mb)));
- ifo_header = (daq_multi_cycle_header_t *)ifo;
- ifo_data = (char *)ifo + sizeof(daq_multi_cycle_header_t);
- // Following line breaks daqd for some reason
- // max_data_size *= 1024 * 1024;
- cycle_data_size = (max_data_size - sizeof(daq_multi_cycle_header_t)) / DAQ_NUM_DATA_BLOCKS_PER_SECOND;
- cycle_data_size -= (cycle_data_size % 8);
- printf ("cycle data size = %d\t%d\n",cycle_data_size, max_data_size_mb);
- sleep(3);
- ifo_header->cycleDataSize = cycle_data_size;
- ifo_header->maxCycle = DAQ_NUM_DATA_BLOCKS_PER_SECOND;
-
- printf("nsys = %d\n",nsys);
- for(ii=0;ii<nsys;ii++) {
- printf("sys %d = %s\n",ii,sname[ii]);
- }
-
- // Make 0MQ socket connections
- for(ii=0;ii<nsys;ii++) {
- // Create a thread to receive data from each data server
- thread_index[ii].index = ii;
- thread_index[ii].src_iface = (niface ? local_iface[ii % niface]: 0);
- pthread_create(&thread_id[ii],NULL,rcvr_thread,(void *)&thread_index[ii]);
- }
-
- if(xmitData) {
- // Connect to Dolphin
- error = dolphin_init();
- printf("Read = 0x%lx \n Write = 0x%lx \n",(long)readAddr,(long)writeAddr);
-
- // Set pointer to xmit header in Dolphin xmit data area.
- mywriteaddr = (char *)writeAddr;
- for(ii=0;ii<IX_BLOCK_COUNT;ii++) {
- xmitHeader[ii] = (daq_multi_cycle_header_t *)mywriteaddr;
- mywriteaddr += IX_BLOCK_SIZE;
- xmitDataOffset[ii] = IX_BLOCK_SIZE * ii + sizeof(struct daq_multi_cycle_header_t);
- }
- }
- //
-
- int nextCycle = 0;
- int64_t prev_sec_and_cycle = 0;
- start_acq = 1;
- int64_t mytime = 0;
- int64_t mylasttime = 0;
- int64_t myptime = 0;
- int64_t n_cycle_time = 0;
- int mytotaldcu = 0;
- char *zbuffer;
- size_t zbuffer_remaining = 0;
- int dc_datablock_size = 0;
- int datablock_size_running = 0;
- int datablock_size_mb_s = 0;
- static const int header_size = sizeof(daq_multi_dcu_header_t);
- char dcstatus[4096];
- char dcs[48];
- int edcuid[10];
- int estatus[10];
- int edbs[10];
- unsigned long ets = 0;
- int dataRdy[MAX_FE_COMPUTERS];
- int any_rdy = 0;
- int sendLength = 0;
-
- int min_cycle_time = 1 << 30;
- int max_cycle_time = 0;
- int mean_cycle_time = 0;
- int pv_dcu_count = 0;
- int pv_total_datablock_size = 0;
- int endpoint_min_count = 1 << 30;
- int endpoint_max_count = 0;
- int endpoint_mean_count = 0;
- int cur_endpoint_ready_count;
- int extra_cycles_min = 1 << 30;
- int extra_cycles_max = 0;
- int extra_cycles_mean = 0;
- int extra_cycles_count = 0;
- char endpoints_missed_buffer[256];
- int endpoints_missed_remaining;
- int missed_flag = 0;
- int missed_nsys[MAX_FE_COMPUTERS];
- //int64_t recv_time[MAX_FE_COMPUTERS];
- //int64_t min_recv_time = 0;
- int64_t cur_ref_time = 0;
- int recv_buckets[(MAX_DELAY_MS/5)+2];
- int entry_binned = 0;
- daq_multi_dcu_data_t* mx_cur_datablock = 0;
-
- mask_t dcu_received_mask;
- mask_t dcu_received_prev_mask;
- mask_t endpoint_received_mask;
- mask_t endpoint_received_prev_mask;
-
- int dcu_endpoint_lookup[DCU_COUNT];
- memset(dcu_endpoint_lookup, 0, sizeof(dcu_endpoint_lookup));
-
- mask_clear(&dcu_received_mask);
- mask_clear(&dcu_received_prev_mask);
- mask_clear(&endpoint_received_mask);
- mask_clear(&endpoint_received_prev_mask);
-
- SimplePV pvs[] = {
- {
- "RECV_BIN_MISS",
- SIMPLE_PV_INT,
- &recv_buckets[0],
- 1,
- -1,
- 1,
- -1,
- },
- {
- "RECV_BIN_0",
- SIMPLE_PV_INT,
- &recv_buckets[1],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_1",
- SIMPLE_PV_INT,
- &recv_buckets[2],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_2",
- SIMPLE_PV_INT,
- &recv_buckets[3],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_3",
- SIMPLE_PV_INT,
- &recv_buckets[4],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_4",
- SIMPLE_PV_INT,
- &recv_buckets[5],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_5",
- SIMPLE_PV_INT,
- &recv_buckets[6],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_6",
- SIMPLE_PV_INT,
- &recv_buckets[7],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_BIN_7",
- SIMPLE_PV_INT,
- &recv_buckets[8],
- nsys,
- -1,
- nsys,
- -1,
- },
- {
- "RECV_BIN_8",
- SIMPLE_PV_INT,
- &recv_buckets[9],
- 1 << 30,
- -1,
- 1 << 30,
- -1,
- },
- {
- "RECV_MIN_MS",
- SIMPLE_PV_INT,
- &min_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MAX_MS",
- SIMPLE_PV_INT,
- &max_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "RECV_MEAN_MS",
- SIMPLE_PV_INT,
- &mean_cycle_time,
-
- 80,
- 45,
- 70,
- 54,
- },
- {
- "DCU_COUNT",
- SIMPLE_PV_INT,
- &pv_dcu_count,
-
- 120,
- 0,
- 115,
- 0,
- },
- {
- "DATA_SIZE",
- SIMPLE_PV_INT,
- &pv_total_datablock_size,
-
- 100*1024*1024,
- 0,
- 90*1024*1024,
- 1*1024*1024,
- },
- {
- "ENDPOINT_MIN_COUNT",
- SIMPLE_PV_INT,
- &endpoint_min_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "ENDPOINT_MAX_COUNT",
- SIMPLE_PV_INT,
- &endpoint_max_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "ENDPOINT_MEAN_COUNT",
- SIMPLE_PV_INT,
- &endpoint_mean_count,
-
- MAX_FE_COMPUTERS,
- 0,
- MAX_FE_COMPUTERS-2,
- 1,
- },
- {
- "DATA_BLOCK_SIZE",
- SIMPLE_PV_INT,
- &dc_datablock_size,
-
- DAQ_DCU_BLOCK_SIZE,
- 0,
- (DAQ_DCU_BLOCK_SIZE*9)/10,
- 1*1024*1024,
-
- },
- {
- "DATA_BLOCK_SIZE_MB_PER_S",
- SIMPLE_PV_INT,
- &datablock_size_mb_s,
-
- DAQ_TRANSIT_MAX_DC_BYTE_SEC/(1024*1024),
- 0,
- (DAQ_TRANSIT_MAX_DC_BYTE_SEC/(1024*1024))*9/10,
- 1,
- },
- {
- "ENDPOINTS_MISSED",
- SIMPLE_PV_STRING,
- endpoints_missed_buffer,
-
- 0, 0, 0, 0,
- },
- {
- "EXTRA_CYCLES_MIN",
- SIMPLE_PV_INT,
- &extra_cycles_min,
-
- 1000,
- 0,
- 1000,
- 5,
- },
- {
- "EXTRA_CYCLES_MAX",
- SIMPLE_PV_INT,
- &extra_cycles_max,
-
- 1000,
- 0,
- 1000,
- 5,
- },
- {
- "EXTRA_CYCLES_MEAN",
- SIMPLE_PV_INT,
- &extra_cycles_mean,
-
- 1000,
- 0,
- 1000,
- 5,
- },
- };
- if (pv_debug_pipe_name)
- {
- pv_debug_pipe = open(pv_debug_pipe_name, O_NONBLOCK | O_RDWR, 0);
- if (pv_debug_pipe < 0) {
- fprintf(stderr, "Unable to open %s for writting (pv status)\n", pv_debug_pipe_name);
- exit(1);
- }
- }
-
- missed_flag = 1;
- memset(&missed_nsys[0], 0, sizeof(missed_nsys));
- memset(recv_buckets, 0, sizeof(recv_buckets));
- do {
- // reset masks
- mask_copy(&dcu_received_prev_mask, &dcu_received_mask);
- mask_copy(&endpoint_received_prev_mask, &endpoint_received_mask);
- mask_clear(&dcu_received_mask);
- mask_clear(&endpoint_received_mask);
-
- any_rdy = wait_for_data(nsys, dataRdy, &nextCycle, &prev_sec_and_cycle, &rcv_errors, &extra_cycles_count);
-
- if (extra_cycles_count < extra_cycles_min)
- {
- extra_cycles_min = extra_cycles_count;
- }
- if (extra_cycles_count > extra_cycles_max)
- {
- extra_cycles_max = extra_cycles_count;
- }
- extra_cycles_mean += extra_cycles_count;
-
- if(any_rdy) {
- // Timing diagnostics
- mytime = s_clock();
- myptime = mytime - mylasttime;
- mylasttime = mytime;
- if (myptime < min_cycle_time) {
- min_cycle_time = myptime;
- }
- if (myptime > max_cycle_time) {
- max_cycle_time = myptime;
- }
- mean_cycle_time += myptime;
- ++n_cycle_time;
- if(do_verbose)printf("Data rdy for cycle = %d\t\t%ld\n",nextCycle,myptime);
-
- // Reset total DCU counter
- mytotaldcu = 0;
- // Reset total DC data size counter
- dc_datablock_size = 0;
- // Get pointer to next data block in shared memory
- nextData = (char *)ifo_data;
- nextData += cycle_data_size * nextCycle;
- ifoDataBlock = (daq_multi_dcu_data_t *)nextData;
- zbuffer = (char *)nextData + header_size;
- zbuffer_remaining = cycle_data_size - header_size;
-
- cur_endpoint_ready_count = 0;
- endpoints_missed_remaining = sizeof(endpoints_missed_buffer);
- endpoints_missed_buffer[0] = '\0';
- //min_recv_time = 0x7fffffffffffffff;
- // Loop over all data buffers received from FE computers
- for(ii=0;ii<nsys;ii++) {
- //recv_time[ii] = -1;
- if(dataRdy[ii]) {
- mask_set_entry(&endpoint_received_mask, ii);
-
- //recv_time[ii] = dataRecvTime[ii];
- //if (recv_time[ii] < min_recv_time) {
- // min_recv_time = recv_time[ii];
- //}
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "+++%s\n", sname[ii]);
- }
- ++cur_endpoint_ready_count;
- int cycle_index = nextCycle % MAX_RECEIVE_BUFFERS;
- mx_cur_datablock = &mxDataBlockSingle[ii][cycle_index];
-
- lock_guard<pthread_spinlock_t> lock_(mxDataBlockSingle_lock[ii][cycle_index]);
- {
- int myc = mx_cur_datablock->header.dcuTotalModels;
- // For each model, copy over data header information
- for (jj = 0; jj < myc; jj++) {
- // Copy data header information
- ifoDataBlock->header.dcuheader[mytotaldcu].dcuId = mx_cur_datablock->header.dcuheader[jj].dcuId;
- ifoDataBlock->header.dcuheader[mytotaldcu].fileCrc = mx_cur_datablock->header.dcuheader[jj].fileCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].status = mx_cur_datablock->header.dcuheader[jj].status;
- ifoDataBlock->header.dcuheader[mytotaldcu].cycle = mx_cur_datablock->header.dcuheader[jj].cycle;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeSec = mx_cur_datablock->header.dcuheader[jj].timeSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].timeNSec = mx_cur_datablock->header.dcuheader[jj].timeNSec;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataCrc = mx_cur_datablock->header.dcuheader[jj].dataCrc;
- ifoDataBlock->header.dcuheader[mytotaldcu].dataBlockSize = mx_cur_datablock->header.dcuheader[jj].dataBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpBlockSize = mx_cur_datablock->header.dcuheader[jj].tpBlockSize;
- ifoDataBlock->header.dcuheader[mytotaldcu].tpCount = mx_cur_datablock->header.dcuheader[jj].tpCount;
-
- mask_set_entry(&dcu_received_mask, ifoDataBlock->header.dcuheader[mytotaldcu].dcuId);
- dcu_endpoint_lookup[ifoDataBlock->header.dcuheader[mytotaldcu].dcuId] = ii;
-
- for (kk = 0; kk < DAQ_GDS_MAX_TP_NUM; kk++)
- ifoDataBlock->header.dcuheader[mytotaldcu].tpNum[kk] = mx_cur_datablock->header.dcuheader[jj].tpNum[kk];
- edbs[mytotaldcu] = mx_cur_datablock->header.dcuheader[jj].tpBlockSize +
- mx_cur_datablock->header.dcuheader[jj].dataBlockSize;
- // Get some diags
- if (ifoDataBlock->header.dcuheader[mytotaldcu].status != 0xbad)
- ets = mx_cur_datablock->header.dcuheader[jj].timeSec;
- estatus[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].status;
- edcuid[mytotaldcu] = ifoDataBlock->header.dcuheader[mytotaldcu].dcuId;
- // Increment total DCU count
- mytotaldcu++;
- }
- // Get the size of the data to transfer
- int mydbs = mx_cur_datablock->header.fullDataBlockSize;
- // Get pointer to data in receive data block
- char *mbuffer = (char *) &mx_cur_datablock->dataBlock[0];
- if (mydbs > zbuffer_remaining) {
- fprintf(stderr,
- "Buffer overflow found. Attempting to write %d bytes to zbuffer which has %d bytes remainging\n",
- (int) mydbs, (int) zbuffer_remaining);
- abort();
- }
- // Copy data from receive buffer to shared memory
- memcpy(zbuffer, mbuffer, mydbs);
- // Increment shared memory data buffer pointer for next data set
- zbuffer += mydbs;
- // Calc total size of data block for this cycle
- dc_datablock_size += mydbs;
- }
- } else {
- missed_nsys[ii] |= missed_flag;
- if (do_verbose && nextCycle == 0) {
- fprintf(stderr, "---%s\n", sname[ii]);
- }
- if (receive_map_dump_file)
- {
- receive_map_dump_counter = 3;
- }
- }
- }
- if (cur_endpoint_ready_count < endpoint_min_count) {
- endpoint_min_count = cur_endpoint_ready_count;
- }
- if (cur_endpoint_ready_count > endpoint_max_count) {
- endpoint_max_count = cur_endpoint_ready_count;
- }
- endpoint_mean_count += cur_endpoint_ready_count;
- // Write total data block size to shared memory header
- ifoDataBlock->header.fullDataBlockSize = dc_datablock_size;
- // Write total dcu count to shared memory header
- ifoDataBlock->header.dcuTotalModels = mytotaldcu;
- // Set multi_cycle head cycle to indicate data ready for this cycle
- ifo_header->curCycle = nextCycle;
- xmitBlockNum = nextCycle % IX_BLOCK_COUNT;
-
- // Calc IX message size
- sendLength = header_size + ifoDataBlock->header.fullDataBlockSize;
- if (nextCycle == 0) {
- memset(recv_buckets, 0, sizeof(recv_buckets));
- }
- /*for (ii = 0; ii < nsys; ++ii) {
- cur_ref_time = 0;
- recv_time[ii] -= min_recv_time;
- entry_binned = 0;
- for (jj = 0; jj < sizeof(recv_buckets)/sizeof(recv_buckets[0]); ++jj) {
- if (recv_time[ii] < cur_ref_time) {
- ++recv_buckets[jj];
- entry_binned = 1;
- break;
- }
- cur_ref_time += 5;
- }
- if (!entry_binned) {
- ++recv_buckets[sizeof(recv_buckets)/sizeof(recv_buckets[0])-1];
- }
- }*/
- datablock_size_running += dc_datablock_size;
- if (nextCycle == 0) {
- datablock_size_mb_s = datablock_size_running / (1024*1024);
- pv_dcu_count = mytotaldcu;
- pv_total_datablock_size = dc_datablock_size;
- mean_cycle_time = (n_cycle_time > 0 ? mean_cycle_time / n_cycle_time : 1 << 31);
- endpoint_mean_count = (n_cycle_time > 0 ? endpoint_mean_count/n_cycle_time : 1<<31);
-
- extra_cycles_mean = (n_cycle_time > 0 ? extra_cycles_count / n_cycle_time : 1 << 31);
-
- endpoints_missed_remaining = sizeof(endpoints_missed_buffer)-1;
- endpoints_missed_buffer[0] = '\0';
- for (ii = 0; ii < sizeof(missed_nsys)/sizeof(missed_nsys[0]) && endpoints_missed_remaining > 0; ++ii) {
- if (missed_nsys[ii]) {
- char tmp[80];
- int count = snprintf(tmp, sizeof(tmp), "%s(%x) ", sname[ii], missed_nsys[ii]);
- if (count < sizeof(tmp)) {
- strncat(endpoints_missed_buffer, tmp, count);
- endpoints_missed_remaining -= count;
- }
- }
- missed_nsys[ii] = 0;
- }
- send_pv_update(pv_debug_pipe, pv_prefix, pvs, sizeof(pvs)/sizeof(pvs[0]));
-
- if (do_verbose) {
- printf("\nData rdy for cycle = %d\t\tTime Interval = %ld msec\n", nextCycle, myptime);
- printf("Min/Max/Mean cylce time %ld/%ld/%ld msec over %ld cycles\n", min_cycle_time, max_cycle_time,
- mean_cycle_time, n_cycle_time);
- printf("Total DCU = %d\t\t\tBlockSize = %d\n", mytotaldcu, dc_datablock_size);
- print_diags(mytotaldcu, nextCycle, sendLength, ifoDataBlock, edbs);
- }
- n_cycle_time = 0;
- min_cycle_time = 1 << 30;
- max_cycle_time = 0;
- mean_cycle_time = 0;
-
- endpoint_min_count = nsys;
- endpoint_max_count = 0;
- endpoint_mean_count = 0;
-
- extra_cycles_min = 1 << 30;
- extra_cycles_max = 0;
- extra_cycles_mean = 0;
-
- missed_flag = 1;
- datablock_size_running = 0;
- } else {
- missed_flag <<= 1;
- }
- if(xmitData) {
- if (sendLength > IX_BLOCK_SIZE)
- {
- fprintf(stderr, "Buffer overflow. Sending %d bytes into a dolphin block that holds %d\n",
- (int)sendLength, (int)IX_BLOCK_SIZE);
- abort();
- }
- // WRITEDATA to Dolphin Network
- SCIMemCpy(sequence,nextData, remoteMap,xmitDataOffset[xmitBlockNum],sendLength,memcpyFlag,&error);
- if (error != SCI_ERR_OK) {
- fprintf(stderr,"SCIMemCpy failed - Error code 0x%x\n",error);
- fprintf(stderr,"For reference the expected error codes are:\n");
- fprintf(stderr,"SCI_ERR_OUT_OF_RANGE = 0x%x\n", SCI_ERR_OUT_OF_RANGE);
- fprintf(stderr,"SCI_ERR_SIZE_ALIGNMENT = 0x%x\n", SCI_ERR_SIZE_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_OFFSET_ALIGNMENT = 0x%x\n", SCI_ERR_OFFSET_ALIGNMENT);
- fprintf(stderr,"SCI_ERR_TRANSFER_FAILED = 0x%x\n", SCI_ERR_TRANSFER_FAILED);
- return error;
- }
- // Set data header information
- xmitHeader[xmitBlockNum]->maxCycle = ifo_header->maxCycle;
- // xmitHeader->cycleDataSize = ifo_header->cycleDataSize;
- xmitHeader[xmitBlockNum]->cycleDataSize = sendLength;;
- // xmitHeader->msgcrc = myCrc;
- // Send cycle last as indication of data ready for receivers
- xmitHeader[xmitBlockNum]->curCycle = ifo_header->curCycle;
- // Have to flush the buffers to make data go onto Dolphin network
- SCIFlush(sequence,SCI_FLAG_FLUSH_CPU_BUFFERS_ONLY);
- }
-
- if (receive_map_dump_counter)
- {
- dump_recv_mask(receive_map_dump_file, nsys, ifoDataBlock->header.dcuheader[0].timeSec, nextCycle, &endpoint_received_mask);
- --receive_map_dump_counter;
- }
- }
- sprintf(dcstatus,"%ld ",ets);
- for(ii=0;ii<mytotaldcu;ii++) {
- sprintf(dcs,"%d %d %d ",edcuid[ii],estatus[ii],edbs[ii]);
- strcat(dcstatus,dcs);
- }
-
- dump_dcu_mask_diff(&dcu_received_prev_mask, &dcu_received_mask, dcu_endpoint_lookup);
- dump_endpoint_mask_diff(&endpoint_received_prev_mask, &endpoint_received_mask);
-
- // Increment cycle count
- nextCycle ++;
- nextCycle %= 16;
- }while (keepRunning); // End of infinite loop
-
- // Stop Rcv Threads
- printf("stopping threads %d \n",nsys);
- stop_working_threads = 1;
-
- // Wait for threads to stop
- sleep(2);
- if(xmitData) {
- printf("closing out ix\n");
- // Cleanup the Dolphin connections
- error = dolphin_closeout();
- }
- printf("closing out zmq\n");
- // Cleanup the ZMQ connections
- for(ii=0;ii<nsys;ii++) {
- if (daq_subscriber[ii]) zmq_close(daq_subscriber[ii]);
- if (daq_context[ii]) zmq_ctx_destroy(daq_context[ii]);
- }
-
- for (ii = 0; ii < MAX_FE_COMPUTERS; ++ii)
- {
- for (jj = 0; jj < MAX_RECEIVE_BUFFERS; ++jj)
- {
- pthread_spin_destroy(&mxDataBlockSingle_lock[ii][jj]);
- }
- }
- exit(0);
-}
diff --git a/src/local_dc/local_dc.c b/src/local_dc/local_dc.c
index 0b928d70d..fc4ef1782 100644
--- a/src/local_dc/local_dc.c
+++ b/src/local_dc/local_dc.c
@@ -63,8 +63,8 @@ int daqStatBit[ 2 ];
void
Usage( )
{
- fprintf( stderr, "Usage of mx_fe:\n" );
- fprintf( stderr, "mx_fe -s <models> <OPTIONS>\n" );
+ fprintf( stderr, "Usage of local_dc:\n" );
+ fprintf( stderr, "local_dc -s <models> <OPTIONS>\n" );
fprintf( stderr,
" -b <buffer> : Name of the mbuf to concentrate the data to "
"locally (defaults to ifo)\n" );
--
GitLab