Adding proxy function and kernel space module using dolphin kernel interfaces

src/dolphin_daemon/Makefile

 
 INCS:= -Iinclude/ -I../include/ -I/opt/DIS/include/ -I/opt/DIS/include/dis
 
+SRCS:= src/main.cpp 
+SRCS+= src/DolphinKernelProxy.cpp
+#SRCS+= src/DolphinNetlinkServer.cpp
+#SRCS+= src/Dolphin_SISCI_Resource.cpp
+
 default:
-	$(CXX) -g src/main.cpp src/DolphinNetlinkServer.cpp src/Dolphin_SISCI_Resource.cpp  -o main $(INCS)  -L/opt/DIS/lib64 -l:libsisci.a -lpthread -lfmt
+	$(CXX) -g $(SRCS)  -o main $(INCS)  -L/opt/DIS/lib64 -l:libsisci.a -lpthread -lfmt
 
 clean:
 	rm -f main

src/dolphin_daemon/include/DolphinKernelProxy.hpp

+#ifndef LIGO_DOLPHIN_KERNEL_PROXY_HPP
+#define LIGO_DOLPHIN_KERNEL_PROXY_HPP
+
+#include "LIGO_Thread.hpp"
+#include "daemon_messages.h"
+#include "NetlinkSocket.hpp"
+
+#include <linux/netlink.h>
+#include <sys/socket.h>
+#include <memory>
+#include <map>
+
+
+class DolphinKernelProxy : public LIGO_Thread
+{
+    public:
+
+        static std::unique_ptr< DolphinKernelProxy > create_instance();
+
+        virtual ~DolphinKernelProxy();
+
+        void thread_body();
+
+        //Initialization Helpers
+        bool init_sockets();
+
+        //Message Handlers 
+        void handle_free_all_req( dolphin_mc_free_all_req * , unsigned bytes );
+        void handle_alloc_req( dolphin_mc_alloc_req * , unsigned bytes);
+
+
+    private:
+
+        DolphinKernelProxy();
+        void build_and_send_alloc_resp_error(DOLPHIN_ERROR_CODES status);
+        void send_netlink_request_response(void * msg_ptr, unsigned sz_bytes);
+
+        //Netlink socket for requests
+        std::unique_ptr< NetlinkSocket > _requests_socket;
+
+
+        //Netlink to dolphin proxy kernel module
+        std::unique_ptr< NetlinkSocket > _dolphin_proxy_socket;
+
+};
+
+
+#endif //LIGO_DOLPHIN_KERNEL_PROXY_HPP

src/dolphin_daemon/include/NetlinkSocket.hpp

+#ifndef LIGO_NETLINK_SOCKET_HPP
+#define LIGO_NETLINK_SOCKET_HPP
+
+//libspdlog-dev
+#include "spdlog/spdlog.h"
+
+
+#include <sys/socket.h>
+#include <linux/netlink.h>
+
+#include <memory>
+
+
+class NetlinkSocket
+{
+    public:
+    
+        static std::unique_ptr< NetlinkSocket > create_instance( int netlnk_link_id, int netlink_groups = 0, double timeout_s = 0.0 )
+        {
+            std::unique_ptr< NetlinkSocket > me_ptr (new NetlinkSocket());
+
+            me_ptr->_sock_fd = socket(PF_NETLINK, SOCK_RAW, netlnk_link_id);
+            if ( me_ptr->_sock_fd < 0) {
+                spdlog::error("socket error: {}", strerror(errno));
+                return nullptr;
+            }
+
+            me_ptr->_my_pid = getpid();  // self pid
+            me_ptr->_netlink_groups = netlink_groups;
+
+            //If netlink_group is set 
+            if( netlink_groups != 0 )
+            {
+                if (setsockopt(me_ptr->_sock_fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, &netlink_groups, sizeof(netlink_groups)) < 0) {
+                    spdlog::error("NetlinkSocket::create_instance() - Cannot add netlink membership for group: {}, error: {}",
+                                  netlink_groups, strerror(errno));
+                    close(me_ptr->_sock_fd);
+                    return nullptr;
+                }
+            }
+
+            //If timeout is set
+            struct timeval timeout;
+            timeout.tv_sec = (int) timeout_s;
+            timeout.tv_usec = (timeout_s - ((int) timeout_s)) * 1e6;
+            if (setsockopt (me_ptr->_sock_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout) < 0)
+            {
+                spdlog::error("NetlinkSocket::create_instance() - Cannot set timeout for NETLINK socket, error: {}",
+                              strerror(errno));
+                close(me_ptr->_sock_fd);
+                return nullptr;
+            }
+
+
+            memset(& me_ptr->_bind_addr, 0, sizeof( me_ptr->_bind_addr));
+            me_ptr->_bind_addr.nl_family = AF_NETLINK;
+            me_ptr->_bind_addr.nl_pid =  me_ptr->_my_pid;
+            me_ptr->_bind_addr.nl_groups = me_ptr->_netlink_groups;  // not in mcast groups 
+            bind( me_ptr->_sock_fd, (struct sockaddr*)& me_ptr->_bind_addr, sizeof( me_ptr->_bind_addr));
+
+
+            memset(& me_ptr->_dest_addr, 0, sizeof( me_ptr->_dest_addr));
+            me_ptr->_dest_addr.nl_family = AF_NETLINK;
+            me_ptr->_dest_addr.nl_pid = 0;   // for kernel 
+            me_ptr->_dest_addr.nl_groups = 0; 
+
+            return me_ptr;
+        }
+
+        ssize_t sendmsg(const void * data, size_t bytes)
+        {
+            struct nlmsghdr *nlh;
+            struct iovec iov;
+            struct msghdr msg;
+
+            //get_msg_buf() stores a buffer for use and reallocates if we need a bigger one
+            nlh = (struct nlmsghdr *) get_msg_buf(NLMSG_SPACE(bytes));
+            nlh->nlmsg_len = NLMSG_SPACE(bytes);
+            nlh->nlmsg_pid = _my_pid;  
+            nlh->nlmsg_flags = 0;
+
+            memcpy(NLMSG_DATA(nlh), data, bytes);
+
+            memset(&iov, 0, sizeof(iov));
+            iov.iov_base = (void *)nlh;
+            iov.iov_len = nlh->nlmsg_len;
+
+            memset(&msg, 0, sizeof(msg));
+            msg.msg_name = (void *)&_dest_addr;
+            msg.msg_namelen = sizeof(_dest_addr);
+            msg.msg_iov = &iov;
+            msg.msg_iovlen = 1;
+
+            return ::sendmsg(_sock_fd, &msg, 0);
+        }
+
+        ssize_t recvmsg(void * data, size_t bytes)
+        {
+            struct iovec iov;
+            struct msghdr msg;
+            struct nlmsghdr *nlh;
+
+            nlh = (struct nlmsghdr *) get_msg_buf(NLMSG_SPACE(bytes));
+            memset(nlh, 0, NLMSG_SPACE(bytes));
+
+            memset(&iov, 0, sizeof(iov));
+            iov.iov_base = (void *)nlh;
+            iov.iov_len = NLMSG_SPACE(bytes);
+
+            memset(&msg, 0, sizeof(msg));
+            msg.msg_name = (void *)&_dest_addr;
+            msg.msg_namelen = sizeof(_dest_addr);
+            msg.msg_iov = &iov;
+            msg.msg_iovlen = 1;
+
+            size_t ret = ::recvmsg(_sock_fd, &msg, 0);
+            if( ret > 0)
+                memcpy(data, NLMSG_DATA(nlh), ret - NLMSG_HDRLEN);
+
+            return ret - NLMSG_HDRLEN;
+        }
+
+
+        virtual ~NetlinkSocket()
+        {
+            close(_sock_fd);
+            if( _msg_buffer ) delete [] _msg_buffer;
+        }
+
+    private:
+
+        NetlinkSocket() = default;
+
+        int                   _sock_fd;
+        struct sockaddr_nl   _bind_addr;
+        struct sockaddr_nl   _dest_addr;
+        int               _netlink_groups;
+
+        int _my_pid;
+        char * _msg_buffer = 0;
+        size_t _msg_buffer_sz = 0;
+
+        char * get_msg_buf(size_t bytes)
+        {
+            if ( _msg_buffer_sz < bytes)
+            {
+                if( _msg_buffer ) delete [] _msg_buffer;
+                _msg_buffer = new char[bytes];
+                memset(_msg_buffer, 0, sizeof(char) * bytes);
+            }
+            return _msg_buffer;
+        }
+
+};
+
+
+
+
+
+#endif //NETLINK_UNICAST_SOCKET_HPP

src/dolphin_daemon/include/daemon_messages.h

@@ -3,27 +3,32 @@
 
 #include "util/fixed_width_types.h"
 
-enum DOLPHIN_DAEMON_MSGS
+typedef enum DOLPHIN_DAEMON_MSGS 
 {
     DOLPHIN_DAEMON_ALLOC_REQ = 1,
     DOLPHIN_DAEMON_ALLOC_RESP = 2,
     DOLPHIN_DAEMON_FREE_REQ = 3
-};
+} DOLPHIN_DAEMON_MSGS;
 
-enum DOLPHIN_ERROR_CODES
+typedef enum DOLPHIN_ERROR_CODES
 {
     DOLPHIN_ERROR_OK = 0,
-    DOLPHIN_ERROR_NO_LIBRARY_SUPPORT = 1,
-    DOLPHIN_ERROR_NO_MCAST_SUPPORT = 2,
-    DOLPHIN_ERROR_SEGMENT_SETUP_ERROR = 3,
-    DOLPHIN_ERROR_SEGMENT_ALREADY_ALLOCATED = 4,
-};
+    DOLPHIN_ERROR_MALFORMED_MSG = 1,
+    DOLPHIN_ERROR_NO_LIBRARY_SUPPORT = 2,
+    DOLPHIN_ERROR_NO_MCAST_SUPPORT = 3,
+    DOLPHIN_ERROR_SEGMENT_SETUP_ERROR = 4,
+    DOLPHIN_ERROR_SEGMENT_ALREADY_ALLOCATED = 5,
+    DOLPHIN_ERROR_KERNEL_MODULE_TIMOUT = 6
+} DOLPHIN_ERROR_CODES;
 
 typedef struct dolphin_mc_header
 {
     uint32_t msg_id;
 } dolphin_mc_header;
 
+//
+// Allocation Request 
+//
 typedef struct dolphin_mc_alloc_req
 {
     dolphin_mc_header header;
@@ -33,6 +38,26 @@ typedef struct dolphin_mc_alloc_req
     uint32_t segment_ids[];
 } dolphin_mc_alloc_req;
 
+static inline int check_alloc_req_valid(dolphin_mc_alloc_req * msg, unsigned payload_len)
+{
+    // Correct msg ID
+    if(msg->header.msg_id != DOLPHIN_DAEMON_ALLOC_REQ) return 0;
+
+    // All required fields are present
+    if(payload_len < sizeof(dolphin_mc_alloc_req)) return 0;
+
+    //Msg is long enough for all requested segments
+    if (payload_len < (msg->num_segments * sizeof(uint32_t) ) 
+                      + sizeof(dolphin_mc_alloc_req) )
+        return 0;
+
+    return 1;
+}
+
+//
+// Allocation response 
+//
+
 typedef struct dolphin_mc_alloc_resp
 {
     dolphin_mc_header header;
@@ -47,11 +72,53 @@ typedef struct dolphin_mc_alloc_resp
     volatile void * addrs[];
 } dolphin_mc_alloc_resp;
 
+static inline int check_alloc_resp_valid(dolphin_mc_alloc_resp * msg, unsigned payload_len)
+{
+    // Correct msg ID
+    if(msg->header.msg_id != DOLPHIN_DAEMON_ALLOC_RESP) return 0;
+
+    // All required fields are present
+    if(payload_len < sizeof(dolphin_mc_alloc_resp)) return 0;
+
+    //Msg is long enough for all requested segments
+    if (payload_len < (msg->num_addrs * sizeof(volatile void *) )
+                      + sizeof(dolphin_mc_alloc_resp) )
+        return 0;
+
+    return 1;
+}
+
+
+//
+// Free all request
+//
 typedef struct dolphin_mc_free_all_req
 {
     dolphin_mc_header header;
 } dolphin_mc_free_all_req;
 
+static inline int check_free_all_valid(dolphin_mc_free_all_req * msg, unsigned payload_len)
+{
+    // Correct msg ID
+    if(msg->header.msg_id != DOLPHIN_DAEMON_FREE_REQ) return 0;
+
+    // All required fields are present
+    if(payload_len < sizeof(dolphin_mc_free_all_req)) return 0;
+
+    return 1;
+}
+
+
+
+//
+// Union for handling any msg type 
+//
+typedef union all_dolphin_daemon_msgs_union
+{
+    dolphin_mc_header * as_hdr;
+    dolphin_mc_alloc_req * as_alloc_req;
+    dolphin_mc_free_all_req * as_free_req;
+} all_dolphin_daemon_msgs_union;
 
 
 #endif //LIGO_DOLPHIN_DAEMON_MESSAGES

src/dolphin_daemon/ks_dolphin/.gitignore

+*.cmd
+*.mod
+*.mod.c
+*.symvers
+*.d
+*.o.d
+temp/

src/dolphin_daemon/ks_dolphin/Makefile

+mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
+mkfile_dir := $(dir $(mkfile_path))
+
+ifndef DOLPHIN_SYM
+DOLPHIN_SYM := $(shell $(mkfile_dir)/../../epics/util/find_module_symvers -q ligo-dolphin-ix)
+endif
+
+KBUILD_EXTRA_SYMBOLS += $(DOLPHIN_SYM)
+
+
+# Set the path to the Kernel build utils.
+KBUILD=/lib/modules/$(shell uname -r)/build/
+
+#Dolphin include directories and compilation flags
+DISDIR = /opt/srcdis
+DOLPHIN_INCS:= -DOS_IS_LINUX=1 -D_DIS_KERNEL_=1 -I$(DISDIR)/src/IRM_GX/drv/src -I$(DISDIR)/src/IRM_GX/drv/src/LINUX -I$(DISDIR)/src/include -I$(DISDIR)/src/include/dis -I$(DISDIR)/src/COMMON/osif/kernel/include -I$(DISDIR)/src/COMMON/osif/kernel/include/LINUX    
+
+#-DDIS_BROADCAST=0x80000000
+
+INCS:=-I$(mkfile_dir)/../include/ -I$(mkfile_dir)/../../include/ $(DOLPHIN_INCS)
+
+obj-m += rts_netlink_dolphin.o
+ccflags-y := -std=gnu99 -Wno-declaration-after-statement $(INCS) 
+
+
+
+all:
+	$(CC) -Wall recv.c -o recv
+	make -C $(KBUILD) M=$(mkfile_dir) modules
+
+clean:
+	rm -f recv
+	make -C $(KBUILD) M=$(mkfile_dir) clean

src/dolphin_daemon/ks_dolphin/recv.c

+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+
+#define MYGRP 17
+
+#define MAX_PAYLOAD 1024  /* maximum payload size*/
+
+
+void read_event(int sock)
+{
+  struct sockaddr_nl nladdr;
+  struct msghdr msg;
+  struct iovec iov;
+  char buffer[65536];
+  int ret;
+
+  iov.iov_base = (void *) buffer;
+  iov.iov_len = sizeof(buffer);
+  msg.msg_name = (void *) &(nladdr);
+  msg.msg_namelen = sizeof(nladdr);
+  msg.msg_iov = &iov;
+  msg.msg_iovlen = 1;
+
+  printf("Listen for message...\n");
+  ret = recvmsg(sock, &msg, 0);
+  if (ret < 0)
+      return;
+
+  char *payload = NLMSG_DATA((struct nlmsghdr *) &buffer);
+  printf("Received from kernel: %s\n", payload);
+}
+
+int main(int argc, char **argv)
+{
+  struct sockaddr_nl src_addr;
+  //struct sockaddr_nl dest_addr;
+  //struct nlmsghdr *nlh;
+  //struct msghdr msg;
+  //struct iovec iov;
+  int sock_fd;
+  //int rc;
+
+
+  sock_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_USERSOCK);
+  if (sock_fd < 0) {
+    printf("socket(): %s\n", strerror(errno));
+    return 1;
+  }
+
+  memset(&src_addr, 0, sizeof(src_addr));
+  src_addr.nl_family = AF_NETLINK;
+  //src_addr.nl_pid = getpid();  /* self pid */
+  src_addr.nl_pid = 0; //from kernel?
+  //src_addr.nl_groups = 0;  /* not in mcast groups */
+  bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));
+
+
+  int group = MYGRP;
+  if (setsockopt(sock_fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, &group, sizeof(group)) < 0) {
+    printf("setsockopt(NETLINK_ADD_MEMBERSHIP): %s\n", strerror(errno));
+    close(sock_fd);
+    return 1;
+  }
+
+
+  while (1) {
+    read_event(sock_fd);
+  }
+
+  close(sock_fd);
+
+  return 0;
+}

src/dolphin_daemon/ks_dolphin/rts_netlink_dolphin.c

+#include <net/sock.h>
+#include <linux/module.h>
+#include <linux/netlink.h>
+#include <linux/skbuff.h>
+#include <linux/kthread.h>  // for threads
+#include <linux/delay.h>
+#include <linux/slab.h>
+
+#include "daemon_messages.h"
+
+#define MYPROTO NETLINK_USERSOCK
+#define MYGRP 17
+
+#define MODULE_NAME "rts_netlink_dolphin"
+
+//Dolphin Includes
+#include "genif.h" //sci_* calls
+#include "id.h" //MULTICAST_NODEID
+//#include "irm_types.h" //Sci_p global
+//#include "priv_io.h" //for chip_type_t, so we can include query.h
+//#include "query.h" //query_mcast_group_size
+
+//Dolphin helper defines
+#define NO_CALLBACK         NULL
+#define NO_FLAGS            0
+#define MAX_MCAST_GROUPS 4 //TODO: Can we query this?
+#define ADAPTER_NUM 0 
+#define MODULE_ID   0
+
+//Dolphin globals
+
+volatile void *         g_read_addrs [ MAX_MCAST_GROUPS ] = {0};
+volatile void *         g_write_addrs [ MAX_MCAST_GROUPS ] = {0};
+sci_l_segment_handle_t  g_segment [ MAX_MCAST_GROUPS ];
+sci_map_handle_t        g_client_map_handle [ MAX_MCAST_GROUPS ];
+sci_r_segment_handle_t  g_remote_segment_handle [ MAX_MCAST_GROUPS ];
+sci_device_info_t       g_sci_dev_info [ MAX_MCAST_GROUPS ];
+
+
+struct sock *nl_sock = NULL;
+
+void send_netlink_response(void * msg, unsigned sz_bytes, unsigned pid)
+{
+    struct sk_buff *skb_out;
+    struct nlmsghdr *nlh;
+
+    skb_out = nlmsg_new(sz_bytes, 0);
+    if (!skb_out) {
+      printk(KERN_ERR MODULE_NAME ": Failed to allocate new skb for response message.\n");
+      return;
+    }
+
+    // put the response into reply
+    nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, sz_bytes, 0);
+    NETLINK_CB(skb_out).dst_group = 0; // not in mcast group 
+    memcpy(nlmsg_data(nlh), msg, sz_bytes);
+
+    //Send response
+    int res = nlmsg_unicast(nl_sock, skb_out, pid);
+    if (res < 0)
+      printk(KERN_ERR MODULE_NAME ": Error while sending response message.\n");
+
+}
+
+void build_and_send_alloc_resp_error(DOLPHIN_ERROR_CODES status, unsigned pid)
+{
+    //Build error response message
+    dolphin_mc_alloc_resp resp;
+    memset(&resp, 0, sizeof(resp));
+    resp.header.msg_id = DOLPHIN_DAEMON_ALLOC_RESP;
+    resp.status = status;
+    resp.num_addrs = 0;
+
+    //No returned segments on error, so msg is just struct size
+    send_netlink_response(&resp, sizeof(resp), pid);
+}
+
+/*
+
+R_OK,
+       SR_DISABLED,
+       SR_WAITING,
+       SR_CHECKING,
+       SR_CHECK_TIMEOUT,
+       SR_LOST,
+       SR_OPEN_TIMEOUT,
+       SR_HEARTBEAT_RECEIVED
+      } session_cb_reason_t;
+*/
+static int32_t
+session_callback( session_cb_arg_t IN arg,
+                  session_cb_reason_t IN reason,
+                  session_cb_status_t IN status,
+                  uint32_t IN target_node,
+                  uint32_t IN local_adapter_number )
+{
+    /// @brief This function contains the required Dolphin callback routine. \n
+    // RTSLOG_INFO("Session callback reason=%d status=%d target_node=%d\n", reason,
+    // status, target_node); if (reason == SR_OK) iop_rfm_valid = 1;
+    
+    /*
+    if ( reason == SR_OK || status == SS_NONE ) //TODO: This does not seem like a good check, we could pass if reason has an error and status is SR_NONE
+        iop_rfm_valid = 1;
+    else
+        iop_rfm_valid = 0;
+        */
+    // This is being called when the one of the other nodes is prepared for
+    // shutdown :TODO: may need to check target_node == <our local node>
+    // if (reason == SR_DISABLED || reason == SR_LOST) iop_rfm_valid = 0;
+    return 0;
+}
+
+/// Function for Dolphin connection callback
+static int32_t
+connect_callback( void IN*               arg,
+                  sci_r_segment_handle_t IN remote_segment_handle,
+                  uint32_t IN reason,
+                  uint32_t IN status )
+{
+    // RTSLOG_INFO("Connect callback reason=%d status=%d\n", reason, status);
+    /*
+    if ( reason == 1 )
+        iop_rfm_valid = 1;
+    if ( reason == 3 )
+        iop_rfm_valid = 0;
+    if ( reason == 5 )
+        iop_rfm_valid = 1;
+        */
+    return 0;
+}
+
+static int32_t
+create_segment_callback( void IN*               arg,
+                         sci_l_segment_handle_t IN local_segment_handle,
+                         uint32_t IN reason,
+                         uint32_t IN source_node,
+                         uint32_t IN local_adapter_number )
+{
+    // RTSLOG_INFO("Create segment callback reason=%d source_node=%d\n", reason,
+    // source_node);
+    return 0;
+}
+
+DOLPHIN_ERROR_CODES allocate_segment(unsigned segment_id, unsigned size)
+{
+    scierror_t err;
+    char*      addr;
+    char*      read_addr;
+    err = sci_create_segment( NO_BINDING,
+                              MODULE_ID,
+                              segment_id,
+                              DIS_BROADCAST,
+                              size,
+                              create_segment_callback,
+                              0,
+                              &g_segment[ segment_id ] );
+    printk(KERN_INFO MODULE_NAME ": DIS segment alloc status %d\n", err);
+    if ( err )
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+
+    err = sci_set_local_segment_available( g_segment[ segment_id ], 0 );
+    printk(KERN_INFO MODULE_NAME ": DIS segment making available status %d\n", err);
+    if ( err )
+    {
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+
+    err = sci_export_segment( g_segment[ segment_id ], 0, DIS_BROADCAST );
+    printk(KERN_INFO MODULE_NAME ": DIS segment export status 0x%x\n", err);
+    if ( err )
+    {
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+
+    g_read_addrs[segment_id] = sci_local_kernel_virtual_address( g_segment[ segment_id ] );
+    if ( read_addr == 0 )
+    {
+        printk(KERN_ERR MODULE_NAME ": DIS sci_local_kernel_virtual_address returned 0\n");
+        sci_unexport_segment( g_segment[ segment_id ], 0, 0);
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+    else
+    {
+        printk(KERN_INFO MODULE_NAME ": Dolphin memory read at 0x%p\n", g_read_addrs[segment_id]);
+
+    }
+    mdelay( 40 );
+
+    err = sci_connect_segment( NO_BINDING,
+                               MULTICAST_NODEID, //broadcast node - should not match any
+                               ADAPTER_NUM,
+                               MODULE_ID,
+                               segment_id,
+                               DIS_BROADCAST,
+                               connect_callback,
+                               0,
+                               &g_remote_segment_handle[ segment_id ] );
+     printk(KERN_INFO MODULE_NAME ": DIS connect segment status %d\n", err);
+    if ( err )
+    {
+        sci_unexport_segment( g_segment[ segment_id ], 0, 0);
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+
+    mdelay( 40 );
+    err = sci_map_segment( g_remote_segment_handle[ segment_id ],
+                           DIS_BROADCAST,
+                           0,
+                           size,
+                           &g_client_map_handle[ segment_id ] );
+     printk(KERN_INFO MODULE_NAME ": DIS segment mapping status 0x%x\n", err);
+    if ( err )
+    {
+        sci_disconnect_segment( &g_remote_segment_handle[ segment_id ], 0 );
+        sci_unexport_segment( g_segment[ segment_id ], 0, 0);
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+
+    g_write_addrs[ segment_id ] = sci_kernel_virtual_address_of_mapping( g_client_map_handle[segment_id ] );
+    if ( addr == 0 )
+    {
+        // RTSLOG_INFO ("Got zero pointer from
+        // sci_kernel_virtual_address_of_mapping\n");
+        sci_disconnect_segment( &g_remote_segment_handle[ segment_id ], 0 );
+        sci_unexport_segment( g_segment[ segment_id ], 0, 0);
+        sci_remove_segment( &g_segment[ segment_id ], 0 );
+        return DOLPHIN_ERROR_SEGMENT_SETUP_ERROR;
+    }
+    else
+    {
+         printk(KERN_INFO MODULE_NAME ": Dolphin memory write at 0x%p\n", addr);
+    }
+
+    sci_register_session_cb( segment_id, 0, session_callback, 0 );
+
+    return DOLPHIN_ERROR_OK;
+}
+
+void free_dolphin_segment( unsigned segment_id )
+{
+    return; //TODO:remove
+
+    sci_unmap_segment( &g_client_map_handle[ segment_id ], 0 );
+    sci_disconnect_segment( &g_remote_segment_handle[ segment_id ], 0 );
+    sci_unexport_segment( g_segment[ segment_id ], 0, 0 );
+    sci_remove_segment( &g_segment[ segment_id ], 0 );
+    sci_cancel_session_cb( segment_id, 0 );
+}
+
+
+void handle_alloc_req( dolphin_mc_alloc_req * req_ptr , unsigned pid)
+{
+    DOLPHIN_ERROR_CODES status;
+
+    //TODO: remove
+    return build_and_send_alloc_resp_error(DOLPHIN_ERROR_SEGMENT_ALREADY_ALLOCATED, pid);
+
+    //Verify the segments are free
+    for(int i =0; i < req_ptr->num_segments; ++i)
+    {
+        if( g_read_addrs[i] != NULL )
+        {
+            printk(KERN_ERR MODULE_NAME ": Request to allocate segment ID: %u, but ID is already allocated.\n", req_ptr->segment_ids[i]);
+            build_and_send_alloc_resp_error(DOLPHIN_ERROR_SEGMENT_ALREADY_ALLOCATED, pid);
+            return;
+        }
+    }
+
+    //Allocate requested segments
+    //
+    for(int i=0; i < req_ptr->num_segments; ++i)
+    {
+        status = allocate_segment(req_ptr->segment_ids[i], req_ptr->segment_sz_bytes);
+
+        if ( status != DOLPHIN_ERROR_OK)
+        {
+            printk(KERN_ERR MODULE_NAME ": Failed to allocate segment ID %u in request.\n", req_ptr->segment_ids[i]);
+            //Found an error, clean up all allocations 
+            for(int j=0; j < i; ++j)
+            {
+                printk(KERN_ERR MODULE_NAME ": Freeing segment %u to clean up partial request.\n", req_ptr->segment_ids[j]);
+                free_dolphin_segment( req_ptr->segment_ids[j] );
+            }
+                
+        }
+
+    }
+
+    //Time to send the accepted respose
+    unsigned num_addrs = req_ptr->num_segments * 2; //One read and write addr for each segment
+    unsigned msg_size = sizeof(dolphin_mc_alloc_resp) + (num_addrs * sizeof(void*)); //msg size is base struct plus the number of addrs we are filling
+
+    dolphin_mc_alloc_resp * resp_ptr = (dolphin_mc_alloc_resp *) kmalloc( msg_size, GFP_KERNEL );
+    resp_ptr->header.msg_id = DOLPHIN_DAEMON_ALLOC_RESP;
+    resp_ptr->status = DOLPHIN_ERROR_OK;
+    resp_ptr->num_addrs = num_addrs;
+
+    int addr_count = 0;
+    for(int i=0; i < req_ptr->num_segments; ++i)
+    {
+        resp_ptr->addrs[addr_count] = g_read_addrs[i];
+        resp_ptr->addrs[addr_count+1] = g_write_addrs[i];
+        addr_count += 2;
+    }
+
+    send_netlink_response(resp_ptr, msg_size, pid);
+    kfree(resp_ptr);
+    printk(KERN_INFO MODULE_NAME ": handle_alloc_req() Finished handling request.\n");
+
+}
+
+void handle_free_all_req( dolphin_mc_free_all_req * req_ptr )
+{
+    for(int i =0; i < MAX_MCAST_GROUPS; ++i)
+    {
+        if ( g_read_addrs[i] != NULL ) //If allocated
+        {
+            free_dolphin_segment( i );
+        }
+    }
+}
+
+static void netlink_recv_unicast_msg(struct sk_buff *skb)
+{
+    struct nlmsghdr *nlh;
+    all_dolphin_daemon_msgs_union any_msg;
+    int pid;
+
+    nlh = (struct nlmsghdr *)skb->data;
+    pid = nlh->nlmsg_pid; // pid of sending process 
+
+    //Make sure we at least have a header before we switch on it
+    if(nlmsg_len(nlh) < sizeof(dolphin_mc_header))
+    {
+        printk(KERN_INFO MODULE_NAME ": netlink_recv_unicast_msg() Malformed message, too short.\n");
+        return;
+    }
+
+    //Switch on message ID and handle accordingly
+    any_msg.as_hdr = (dolphin_mc_header *) nlmsg_data(nlh);
+    switch ( any_msg.as_hdr->msg_id )
+    {
+        case DOLPHIN_DAEMON_ALLOC_REQ:
+
+            if ( check_alloc_req_valid(any_msg.as_alloc_req, nlmsg_len(nlh)) == 0)
+            {
+                printk(KERN_INFO MODULE_NAME ": netlink_recv_unicast_msg() Malformed DOLPHIN_DAEMON_ALLOC_REQ\n");
+                break;
+            }
+            handle_alloc_req( any_msg.as_alloc_req, pid );
+
+        break;
+        case DOLPHIN_DAEMON_FREE_REQ:
+            handle_free_all_req( any_msg.as_free_req );
+        break;
+        default:
+            printk(KERN_INFO MODULE_NAME ": netlink_recv_unicast_msg() - Invalid message id of %u, discarding message.", any_msg.as_hdr->msg_id);
+        break;
+
+    }
+
+}
+
+
+
+
+
+static int __init rts_netlink_dolphin_init(void)
+{
+    printk(KERN_INFO MODULE_NAME ": Init module\n");
+
+    //scierror_t error;
+    //query_mcast_group_size(Sci_p, &g_mcast_group_sz); //Where do we get Sci_p from
+
+    struct netlink_kernel_cfg cfg = {
+        .input = netlink_recv_unicast_msg,
+    };
+
+    nl_sock = netlink_kernel_create(&init_net, MYPROTO, &cfg);
+    if (!nl_sock) {
+        printk(KERN_ALERT MODULE_NAME ": Error creating netlink socket.\n");
+        return -10;
+    }
+
+    return 0;
+}
+
+static void __exit rts_netlink_dolphin_exit(void)
+{
+    printk(KERN_INFO MODULE_NAME ": Exit module\n");
+
+    netlink_kernel_release(nl_sock);
+}
+
+module_init(rts_netlink_dolphin_init);
+module_exit(rts_netlink_dolphin_exit);
+
+MODULE_LICENSE("Dual MIT/GPL");

src/dolphin_daemon/src/DolphinKernelProxy.cpp

+#include "DolphinKernelProxy.hpp"
+
+#include "Dolphin_SISCI_Resource.hpp"
+
+//libspdlog-dev
+#include "spdlog/spdlog.h"
+#include "spdlog/cfg/env.h"
+
+#include <vector>
+
+//Netlink Defines
+#define NETLINK_GROUP_NUM 17
+#define MAX_PAYLOAD 4096
+
+std::unique_ptr<DolphinKernelProxy> DolphinKernelProxy::create_instance()
+{
+    std::unique_ptr<DolphinKernelProxy> me_ptr (new DolphinKernelProxy());
+
+    me_ptr->_requests_socket = NetlinkSocket::create_instance(NETLINK_USERSOCK, NETLINK_GROUP_NUM, 1.0);
+    if( !me_ptr->_requests_socket ) return nullptr;
+
+    me_ptr->_dolphin_proxy_socket = NetlinkSocket::create_instance(NETLINK_USERSOCK, 0, 3.0);
+    if( !me_ptr->_dolphin_proxy_socket ) return nullptr;
+
+
+    return me_ptr;
+}
+
+DolphinKernelProxy::~DolphinKernelProxy()
+{
+}
+
+DolphinKernelProxy::DolphinKernelProxy()
+{
+}
+
+void DolphinKernelProxy::thread_body()
+{
+
+    int ret;
+    char buffer[MAX_PAYLOAD] = {0};
+
+    while( !should_stop() )
+    {
+        ret = _requests_socket->recvmsg(buffer, MAX_PAYLOAD);
+
+        if(ret == -1) continue; //Timout
+
+        if (ret < 0)
+        {
+            spdlog::warn("DolphinKernelProxy - Error {}, when receiving netlink multicast message from group {}", 
+                         ret, NETLINK_GROUP_NUM);
+            continue;
+        }
+
+
+        unsigned payload_sz =  ret - NLMSG_HDRLEN ;
+        spdlog::info("Received {} bytes, payload_sz : {}", ret, payload_sz);
+
+        all_dolphin_daemon_msgs_union any_msg;
+        any_msg.as_hdr = (dolphin_mc_header *) NLMSG_DATA((struct nlmsghdr *) &buffer);;
+        
+
+        switch ( any_msg.as_hdr->msg_id )
+        {
+            case DOLPHIN_DAEMON_ALLOC_REQ:
+
+                //Verify message length
+                if ( check_alloc_req_valid(any_msg.as_alloc_req, payload_sz) == 0)
+                {
+                    spdlog::error("DolphinKernelProxy - Invalid message size {}, for num_segments {}", 
+                                  payload_sz, any_msg.as_alloc_req->num_segments);
+                    build_and_send_alloc_resp_error(DOLPHIN_ERROR_MALFORMED_MSG);
+                    break;
+                }
+                handle_alloc_req( any_msg.as_alloc_req, payload_sz );
+
+            break;
+            case DOLPHIN_DAEMON_FREE_REQ:
+                handle_free_all_req( any_msg.as_free_req, payload_sz );
+            break;
+            default:
+                spdlog::error("DolphinKernelProxy - Invalid message id of {}, discarding message.", any_msg.as_hdr->msg_id);
+                build_and_send_alloc_resp_error(DOLPHIN_ERROR_MALFORMED_MSG);
+                continue;
+            break;
+
+        }
+
+        spdlog::info("DolphinKernelProxy - Received kernel multicast message.");
+
+    }
+
+
+
+}
+
+void DolphinKernelProxy::handle_free_all_req( dolphin_mc_free_all_req * req_ptr, unsigned bytes)
+{
+
+    //Proxy the request to the kernel module
+    _dolphin_proxy_socket->sendmsg(req_ptr, bytes);
+    spdlog::info("DolphinKernelProxy::handle_free_all_req() - Dolphin resources released.");
+
+    return;
+}
+
+
+void DolphinKernelProxy::handle_alloc_req( dolphin_mc_alloc_req * req_ptr, unsigned bytes )
+{
+    DOLPHIN_ERROR_CODES status;
+    char buffer[MAX_PAYLOAD] = {0};
+
+    spdlog::info("dolphin_mc_alloc_req message received.");
+    spdlog::info("There are {} num_segments requested.", req_ptr->num_segments);
+
+    for(int i =0; i < req_ptr->num_segments; ++i)
+    {
+        spdlog::info("Segment {}", req_ptr->segment_ids[i]);
+    }
+
+
+    //Proxy the request to the kernel module
+    _dolphin_proxy_socket->sendmsg(req_ptr, bytes);
+    int ret = _dolphin_proxy_socket->recvmsg(buffer, MAX_PAYLOAD);
+    if(ret == -1) //Timeout
+    {
+        spdlog::error("The dolphin kernel space module did not respond to the proxied dolphin_mc_alloc_req.");
+        build_and_send_alloc_resp_error(DOLPHIN_ERROR_KERNEL_MODULE_TIMOUT);
+    }
+
+
+    send_netlink_request_response(buffer, ret);
+
+
+    return;
+    
+}
+
+
+void DolphinKernelProxy::build_and_send_alloc_resp_error(DOLPHIN_ERROR_CODES status)
+{
+    //Build error response message
+    dolphin_mc_alloc_resp resp;
+    memset(&resp, 0, sizeof(resp));
+    resp.header.msg_id = DOLPHIN_DAEMON_ALLOC_RESP;
+    resp.status = status;
+    resp.num_addrs = 0;
+
+    //No returned segments on error, so msg is just struct size
+    send_netlink_request_response(&resp, sizeof(resp));
+}
+
+void DolphinKernelProxy::send_netlink_request_response(void * msg_ptr, unsigned sz_bytes)
+{
+   _requests_socket->sendmsg(msg_ptr, sz_bytes); 
+}
+

src/dolphin_daemon/src/DolphinNetlinkServer.cpp

@@ -65,35 +65,33 @@ void DolphinNetlinkServer::thread_body()
         unsigned payload_sz =  ret - NLMSG_HDRLEN ;
         spdlog::info("Received {} bytes, payload_sz : {}", ret, payload_sz);
 
-        union msgs_t
-        {
-            dolphin_mc_header * hdr;
-            dolphin_mc_alloc_req * alloc_req; 
-            dolphin_mc_free_all_req * free_req;
-        };
-        msgs_t msg;
-        msg.hdr = (dolphin_mc_header *) NLMSG_DATA((struct nlmsghdr *) &buffer);;
+        all_dolphin_daemon_msgs_union any_msg;
+        any_msg.as_hdr = (dolphin_mc_header *) NLMSG_DATA((struct nlmsghdr *) &buffer);;
         
 
-        switch ( msg.hdr->msg_id )
+        switch ( any_msg.as_hdr->msg_id )
         {
             case DOLPHIN_DAEMON_ALLOC_REQ:
 
                 //Verify message length
-                if (payload_sz < (msg.alloc_req->num_segments * sizeof(uint32_t) ) + sizeof(dolphin_mc_alloc_req) )
+                if ( check_alloc_req_valid(any_msg.as_alloc_req, payload_sz) == 0)
                 {
                     spdlog::error("DolphinNetlinkServer - Invalid message size {}, for num_segments {}", 
-                                  payload_sz, msg.alloc_req->num_segments);
-                    continue;
+                                  payload_sz, any_msg.as_alloc_req->num_segments);
+                    build_and_send_alloc_resp_error(DOLPHIN_ERROR_MALFORMED_MSG);
+                    break;
                 }
-                handle_alloc_req( msg.alloc_req );
+                handle_alloc_req( any_msg.as_alloc_req );
 
             break;
             case DOLPHIN_DAEMON_FREE_REQ:
-                handle_free_all_req( msg.free_req );
+
+
+                handle_free_all_req( any_msg.as_free_req );
             break;
             default:
-                spdlog::error("DolphinNetlinkServer - Invalid message id of {}, discarding message.", msg.hdr->msg_id);
+                spdlog::error("DolphinNetlinkServer - Invalid message id of {}, discarding message.", any_msg.as_hdr->msg_id);
+                build_and_send_alloc_resp_error(DOLPHIN_ERROR_MALFORMED_MSG);
                 continue;
             break;
 
@@ -179,7 +177,7 @@ void DolphinNetlinkServer::handle_alloc_req( dolphin_mc_alloc_req * req_ptr )
 
     send_netlink_message(resp_ptr, msg_size);
 
-
+    free(resp_ptr);
 
     return;
     
@@ -237,6 +235,7 @@ void DolphinNetlinkServer::send_netlink_message(void * msg_ptr, unsigned sz_byte
     free(nlh);
 }
 
+
 bool DolphinNetlinkServer::init_sockets()
 {
     //
@@ -280,4 +279,3 @@ bool DolphinNetlinkServer::init_sockets()
 }
 
 
-

src/dolphin_daemon/src/main.cpp

@@ -4,6 +4,7 @@
 
 #include "LIGO_Thread.hpp"
 #include "DolphinNetlinkServer.hpp"
+#include "DolphinKernelProxy.hpp"
 
 using namespace std;
 
@@ -32,7 +33,8 @@ int main(int argc, char * argv[])
 {
     signal(SIGINT, signal_handler);
 
-    std::unique_ptr<DolphinNetlinkServer> tt = DolphinNetlinkServer::create_instance();
+    //std::unique_ptr<DolphinNetlinkServer> tt = DolphinNetlinkServer::create_instance();
+    std::unique_ptr< DolphinKernelProxy > tt = DolphinKernelProxy::create_instance();
     if(tt.get() == nullptr) return 0;
 
     tt->nonblocking_start();

src/pcie_switch/kmod/Makefile

@@ -22,7 +22,7 @@ EXTRA_CFLAGS += -O3
 
 #Dolphin include directories and compilation flags
 DISDIR = /opt/srcdis
-EXTRA_CFLAGS += -DOS_IS_LINUX=1 -D_DIS_KERNEL_=1 -I$(DISDIR)/src/IRM_GX/drv/src -I$(DISDIR)/src/IRM_GX/drv/src/LINUX -I$(DISDIR)/src/include -I$(DISDIR)/src/include/dis -I$(DISDIR)/src/COMMON/osif/kernel/include -I$(DISDIR)/src/COMMON/osif/kernel/include/LINUX -I$M/../../drv/ -DDOLPHIN_TEST=1  -DDIS_BROADCAST=0x80000000
+EXTRA_CFLAGS += -DOS_IS_LINUX=1 -D_DIS_KERNEL_=1 -I$(DISDIR)/src/IRM_GX/drv/src -I$(DISDIR)/src/IRM_GX/drv/src/LINUX -I$(DISDIR)/src/include -I$(DISDIR)/src/include/dis -I$(DISDIR)/src/COMMON/osif/kernel/include -I$(DISDIR)/src/COMMON/osif/kernel/include/LINUX -I$M/../../drv/ -DDOLPHIN_TEST=1  #-DDIS_BROADCAST=0x80000000
 
 # Set the path to the Kernel build utils.
 KBUILD=/lib/modules/$(shell uname -r)/build/