Adding atomic vars to signal kernel models should exit

src/fe/controllerApp.c

@@ -36,7 +36,7 @@
 #include "modelRateInfo.h"
 #include "fm10Gen.h"
 #include "util/printl.h"
-
+#include "util/kernel/exit_signaling.h"
 #include "../fe/timing_common.h" //captureEocTiming
 #include "../fe/timing_kernel.h"
 
@@ -204,6 +204,8 @@ fe_start_controller( void )
     if ( initVars( pDsp[ 0 ], pDsp[ 0 ], dspCoeff, MAX_MODULES, pCoeff[ 0 ] ) )
     {
         pLocalEpics->epicsOutput.fe_status = FILT_INIT_ERROR;
+        wait_for_exit_signal();
+        atomic_set(&g_atom_has_exited, 1);
         return;
     }
 
@@ -243,6 +245,8 @@ fe_start_controller( void )
     {
         pLocalEpics->epicsOutput.fe_status = DAQ_INIT_ERROR;
         vmeDone = 1;
+        wait_for_exit_signal();
+        atomic_set(&g_atom_has_exited, 1);
         return;
     }
 #endif
@@ -287,6 +291,8 @@ fe_start_controller( void )
     if ( status )
     {
         pLocalEpics->epicsOutput.fe_status = DAC_INIT_ERROR;
+        wait_for_exit_signal();
+        atomic_set(&g_atom_has_exited, 1);
         return;
     }
 
@@ -385,6 +391,8 @@ fe_start_controller( void )
                 pLocalEpics->epicsOutput.diagWord |= ADC_TIMEOUT_ERR;
                 pLocalEpics->epicsOutput.fe_status = ADC_TO_ERROR;
                 deallocate_dac_channels( );
+                wait_for_exit_signal();
+                atomic_set(&g_atom_has_exited, 1);
                 return;
             }
 
@@ -497,7 +505,7 @@ fe_start_controller( void )
         // *****************************************************************
         /// \> Check if code exit is requested
         if ( cycleNum == MAX_MODULES )
-            vmeDone = stop_working_threads |
+            vmeDone =  atomic_read(&g_atom_should_exit) |
                 checkEpicsReset( cycleNum, (struct CDS_EPICS*)pLocalEpics );
         // *****************************************************************
 
@@ -673,5 +681,6 @@ fe_start_controller( void )
     deallocate_dac_channels( );
 
     /* System reset command received */
+    atomic_set(&g_atom_has_exited, 1);
     return;
 }

src/fe/controllerIop.c

@@ -37,6 +37,7 @@
 #include "fm10Gen_types.h"
 #include "controller.h"
 #include "modelRateInfo.h"
+#include "util/kernel/exit_signaling.h"
 #include "drv/daqLib.h"
 #include "../fe/timing_kernel.h"
 #include "../fe/sync21pps.h"
@@ -256,6 +257,8 @@ fe_start_controller( void )
     {
         pLocalEpics->epicsOutput.fe_status = FILT_INIT_ERROR;
         fe_status_return = FILT_INIT_ERROR;
+        wait_for_exit_signal();
+        atomic_set(&g_atom_has_exited, 1);
         return;
     }
 
@@ -291,6 +294,8 @@ fe_start_controller( void )
         pLocalEpics->epicsOutput.fe_status = DAQ_INIT_ERROR;
         fe_status_return = DAQ_INIT_ERROR;
         vmeDone = 1;
+        wait_for_exit_signal();
+        atomic_set(&g_atom_has_exited, 1);
         return;
     }
 
@@ -619,7 +624,7 @@ fe_start_controller( void )
             pLocalEpics->epicsOutput.diagWord |= ADC_TIMEOUT_ERR;
             pLocalEpics->epicsOutput.fe_status = ADC_TO_ERROR;
             pLocalEpics->epicsOutput.epicsSync++;
-            stop_working_threads = 1;
+            atomic_set(&g_atom_should_exit, 1);
             vmeDone = 1;
             continue;
         }
@@ -631,7 +636,7 @@ fe_start_controller( void )
             feStatus |= adc_status_update( &adcinfo );
             pLocalEpics->epicsOutput.stateWord = FE_ERROR_ADC;
             pLocalEpics->epicsOutput.fe_status = CHAN_HOP_ERROR;
-            stop_working_threads = 1;
+            atomic_set(&g_atom_should_exit, 1);
             vmeDone = 1;
             pLocalEpics->epicsOutput.epicsSync++;
             continue;
@@ -884,7 +889,7 @@ fe_start_controller( void )
             // *****************************************************************
             /// \> Check if code exit is requested
             if ( cycleNum == MAX_MODULES )
-                vmeDone = stop_working_threads |
+                vmeDone = atomic_read(&g_atom_should_exit) |
                     checkEpicsReset( cycleNum, (struct CDS_EPICS*)pLocalEpics );
 
             // *****************************************************************
@@ -1249,5 +1254,6 @@ fe_start_controller( void )
     pLocalEpics->epicsOutput.cpuMeter = 0;
 
     /* System reset command received */
+    atomic_set(&g_atom_has_exited, 1);
     return;
 }

src/fe/moduleLoad.c

@@ -8,6 +8,8 @@
 #include "verify_card_count.h"
 #include "print_io_info.h"
 #include "util/printl.h"
+#include "util/timing.h"
+#include "util/kernel/exit_signaling.h"
 #include "drv/map.h" //mapPciModules()
 #include "drv/ligoPcieTiming.h"
 #include "../fe/verify_slots.h"
@@ -29,10 +31,18 @@ extern void  fe_start_controller( void );
 //
 // File function prototypes
 //
-void rt_fe_cleanup( void );
-int rt_fe_init( void );
+static void rt_fe_cleanup( void );
+static int rt_fe_init( void );
+
+//
+// Signaling variables for proper module shutdown logic
+atomic_t g_atom_should_exit = ATOMIC_INIT(0);
+atomic_t g_atom_has_exited = ATOMIC_INIT(0);
 
 
+//
+// This symbol is used to enforce the IOP model
+// is first loaded, before any app models
 #ifdef IOP_MODEL
 int need_to_load_IOP_first;
 EXPORT_SYMBOL( need_to_load_IOP_first );
@@ -55,14 +65,13 @@ static int  fe_start_controller_kthread( void * arg )
     fe_start_controller();
     return 0;
 }
-
 #endif //NO_CPU_SHUTDOWN
 
-// MAIN routine: Code starting point
+
+// Linux Module init: Code starting point
 // ****************************************************************
 /// Startup function for initialization of kernel module.
-int
-rt_fe_init( void )
+static int __init rt_fe_init( void )
 {
     int jj, kk; /// @param ii,jj,kk default loop counters
     int cards; /// @param cards Number of PCIe cards found on bus
@@ -85,7 +94,7 @@ rt_fe_init( void )
 
 #ifndef NO_CPU_SHUTDOWN
     /// Verify requested core is free.
-    if ( is_cpu_taken_by_rcg_model( CPUID ) )
+    if ( is_cpu_occupied( CPUID ) )
     {
         printl( KERN_ALERT "Error: CPU %d already taken\n", CPUID );
         return -1;
@@ -141,7 +150,6 @@ rt_fe_init( void )
     {
         pLocalEpics->epicsOutput.fe_status = IO_CARD_MAP_ERROR;
         rt_fe_cleanup( );
-        detach_shared_memory();
         return -5;
     }
 
@@ -151,7 +159,6 @@ rt_fe_init( void )
         printl( "" SYSTEM_NAME_STRING_LOWER
                 ": ERROR: No ADC cards found - exiting\n" );
         rt_fe_cleanup( );
-        detach_shared_memory();
         return -5;
     }
 
@@ -231,7 +238,6 @@ rt_fe_init( void )
         printl( "" SYSTEM_NAME_STRING_LOWER
                 ": ERROR: Exit on incorrect card count \n" );
         rt_fe_cleanup( );
-        detach_shared_memory();
         return -5;
     }
 #endif
@@ -262,7 +268,7 @@ rt_fe_init( void )
     printl( "" SYSTEM_NAME_STRING_LOWER ": Locking CPU core %d\n", CPUID );
 
     // The code runs on the disabled CPU
-    set_fe_code_idle( fe_start_controller, CPUID );
+    set_rt_callback( fe_start_controller, CPUID );
     msleep( 100 );
     rts_isolator_exec( CPUID );
 
@@ -270,40 +276,49 @@ rt_fe_init( void )
     return 0;
 }
 
+void wait_for_module_exit(void)
+{
+    uint64_t stop_sig_time_ns = getMonotonic_ns_utin64();
+    if( is_cpu_occupied(CPUID) )
+    {
+        set_rt_callback( 0, CPUID );
+        
+        // Wait for the module to signal that it has exited
+        while (atomic_read(&g_atom_has_exited) == 0)
+        {
+            msleep( 1 );
+        }
+
+        printl("It took %lld ms for the RT code to exit.\n",
+                 (getMonotonic_ns_utin64() - stop_sig_time_ns)/1000000);
+
+        set_rt_callback( 0, CPUID );
+        msleep( 1000 );
+
+        // Bring the CPU back up
+        rts_isolator_cleanup( CPUID );
+    }
+}
+
 /// Kernel module cleanup function
-void
-rt_fe_cleanup( void )
+static void __exit rt_fe_cleanup( void )
 {
-#ifndef NO_CPU_SHUTDOWN
-    /// Unset the code callback
-    set_fe_code_idle( 0, CPUID );
-#endif
 
-    // printl("Setting stop_working_threads to 1\n");
-    // Stop the code and wait
+    // Signal the model to stop
+    atomic_set(&g_atom_should_exit, 1);
+
 #ifdef NO_CPU_SHUTDOWN
     kthread_stop( sthread );
-#endif
-    stop_working_threads = 1;
     msleep( 1000 );
+#else
+    wait_for_module_exit();
+#endif
 
 #ifdef DOLPHIN_TEST
     /// Cleanup Dolphin card connections
     finish_dolphin( );
 #endif
 
-#ifndef NO_CPU_SHUTDOWN
-
-    /// Bring the CPU core back on line
-    // Unset the code callback
-    set_fe_code_idle( 0, CPUID );
-    // printll("Will bring back CPU %d\n", CPUID);
-    msleep( 1000 );
-    // Bring the CPU back up
-    rts_isolator_cleanup( CPUID );
-    msleep( 1000 );
-#endif
-
     // Print out any error messages from FE code on exit
     print_exit_messages( fe_status_return, fe_status_return_subcode, SYSTEM_NAME_STRING_LOWER );
     detach_shared_memory( );

src/include/util/kernel/exit_signaling.h

+#ifndef LIGO_EXIT_SIGNALING_H
+#define LIGO_EXIT_SIGNALING_H
+
+#include <linux/types.h>
+#include <linux/delay.h>
+#include <linux/atomic.h>
+
+// Used to signal the model to exit, defined in fe/moduleLoad.c
+extern atomic_t g_atom_should_exit;
+// Used by fe_start_controller() (The model) to signal it has exited,
+// defined in fe/moduleLoad.c
+extern atomic_t g_atom_has_exited; 
+
+// When a model experiences a nonrecoverable error and needs to exit
+// we block execution with this function until the model is rmmod-ed
+static void wait_for_exit_signal(void)
+{
+    while(atomic_read(&g_atom_should_exit) == 0)
+    {
+    }
+}
+
+
+#endif //LIGO_EXIT_SIGNALING_H