Provide Interface for NF to get TX/RX Rings (#164)

This provides a new interface for NFs to get data from the manager.  Instead of being limited to the traditional model where an NF asks the ONVM manager for packets, this addition enables an NF to have direct access to its TX/RX rings and its stat's structure.

Commit log:
  * Extend nflib NF API to accomodate Snort DAQ NF requirements.
  * Only allow a single NF Mode
  * Update NF dev documentation about dual interfaces
  * Give SpeedTester option to run with direct rings

docs/NF_Dev.md

@@ -20,6 +20,9 @@ The first function, `int onvm_nf_init(int argc, char *argv[], struct onvm_nf_inf
 
 The second function, `int onvm_nf_run(struct onvm_nf_info* info, void(*handler)(struct rte_mbuf* pkt, struct onvm_pkt_meta* meta))`, is the communication protocol between NF and manager where the NF is providing a packet handler function pointer to the manager.  The manager uses this function pointer to pass packets to when they arrive for a specific NF.  It continuously loops and listens for new packets and passes them on to the packet handler function.
 
+### Advanced Ring Manipulation
+For advanced NFs, calling `onvm_nf_run` (as described above) is actually optional. There is a second mode where NFs can interface directly with the shared data structures. Be warned that using this interface means the NF is responsible for its own packets, and the NF Guest Library can make fewer guarantees about overall system performance. Additionally, the NF is responsible for maintaining its own statistics. An advanced NF can call `onvm_nflib_get_rx_ring(struct onvm_nf_info *info)` or `onvm_nflib_get_tx_ring(struct onvm_nf_info *info)` to get the `struct rte_ring *` for RX and TX, respectively. NFs can also call `onvm_nflib_get_tx_stats(struct onvm_nf_info *info)` to get a reference to `struct client_tx_stats *`. Finally, note that using any of these functions precludes you from calling `onvm_nf_run`, and calling `onvm_nf_run` precludes you from calling any of these advanced functions (they will return `NULL`). The first interface you use if the one you get.
+
 TCP/IP UDP Library
 --
 
@@ -66,4 +69,4 @@ Controlling packets in openNetVM is done with actions.  There are four actions,
 [./onvm/shared/onvm_pkt_helper.h]: https://github.com/sdnfv/openNetVM/blob/master/onvm/shared/onvm_pkt_helper.h#L44
 [./onvm/shared/onvm_pkt_helper.h]: https://github.com/sdnfv/openNetVM/blob/master/onvm/shared/onvm_pkt_helper.h#L56
 [./onvm/shared/onvm_pkt_helper.h]: https://github.com/sdnfv/openNetVM/blob/master/onvm/shared/onvm_pkt_helper.h#L68
-[./openNetVM/onvm/shared/common.h]: https://github.com/sdnfv/openNetVM/blob/master/onvm/shared/common.h#L28
+[./openNetVM/onvm/shared/common.h]: https://github.com/sdnfv/openNetVM/blob/master/onvm/shared/common.h#L28

examples/speed_tester/go.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
 function usage {
-        echo "$0 CPU-LIST SERVICE-ID DST [-p PRINT] [-n NF-ID]"
+        echo "$0 CPU-LIST SERVICE-ID DST [-p PRINT] [-n NF-ID] [-a]"
         echo "$0 3,7,9 1 2 --> cores 3,7, and 9, with Service ID 1, and forwards to service ID 2"
         echo "$0 3,7,9 1 2 1000 --> cores 3,7, and 9, with Service ID 1, forwards to service ID 2,  and Print Rate of 1000"
+        echo "Pass '-a' to signal the NF to use advanced ring manipulation"
         exit 1
 }
 
@@ -20,13 +21,14 @@ then
     usage
 fi
 
-while getopts ":p:n:" opt; do
+while getopts ":p:n:a" opt; do
   case $opt in
     p) print="-p $OPTARG";;
     n) instance="-n $OPTARG";;
+    a) rings="-a true";;
     \?) echo "Unknown option -$OPTARG" && usage
     ;;
   esac
 done
 
-exec sudo $SCRIPTPATH/build/speed_tester -l $cpu -n 3 --proc-type=secondary -- -r $service $instance -- -d $dst $print
+exec sudo $SCRIPTPATH/build/speed_tester -l $cpu -n 3 --proc-type=secondary -- -r $service $instance -- -d $dst $print $rings

examples/speed_tester/speed_tester.c

@@ -48,12 +48,14 @@
 #include <stdlib.h>
 #include <getopt.h>
 #include <string.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_mbuf.h>
 #include <rte_ip.h>
 #include <rte_mempool.h>
 #include <rte_cycles.h>
+#include <rte_ring.h>
 
 
 #include "onvm_nflib.h"
@@ -63,20 +65,23 @@
 
 #define NUM_PKTS 128
 #define PKTMBUF_POOL_NAME "MProc_pktmbuf_pool"
+#define PKT_READ_SIZE  ((uint16_t)32)
 
 /* Struct that contains information about this NF */
 struct onvm_nf_info *nf_info;
 
 /* number of package between each print */
 static uint32_t print_delay = 10000000;
 static uint16_t destination;
+static uint8_t use_direct_rings = 0;
+static uint8_t keep_running = 1;
 
 /*
  * Print a usage message
  */
 static void
 usage(const char *progname) {
-        printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d <destination> -p <print_delay>\n\n", progname);
+        printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d <destination> -p <print_delay> -a <use_advanced_rings>\n\n", progname);
 }
 
 /*
@@ -86,8 +91,11 @@ static int
 parse_app_args(int argc, char *argv[], const char *progname) {
         int c, dst_flag = 0;
 
-        while ((c = getopt (argc, argv, "d:p:")) != -1) {
+        while ((c = getopt (argc, argv, "d:p:a:")) != -1) {
                 switch (c) {
+                case 'a':
+                        use_direct_rings = 1;
+                        break;
                 case 'd':
                         destination = strtoul(optarg, NULL, 10);
                         dst_flag = 1;
@@ -174,6 +182,66 @@ packet_handler(struct rte_mbuf* pkt, struct onvm_pkt_meta* meta) {
 }
 
 
+static void
+handle_signal(int sig)
+{
+        if (sig == SIGINT || sig == SIGTERM)
+                keep_running = 0;
+}
+
+
+static void
+run_advanced_rings(void) {
+        void *pkts[PKT_READ_SIZE];
+        struct onvm_pkt_meta* meta;
+        uint16_t i, j, nb_pkts;
+        void *pktsTX[PKT_READ_SIZE];
+        int tx_batch_size;
+        struct rte_ring *rx_ring;
+        struct rte_ring *tx_ring;
+        volatile struct client_tx_stats *tx_stats;
+
+        printf("Process %d handling packets using advanced rings\n", nf_info->instance_id);
+        printf("[Press Ctrl-C to quit ...]\n");
+
+        /* Listen for ^C and docker stop so we can exit gracefully */
+        signal(SIGINT, handle_signal);
+        signal(SIGTERM, handle_signal);
+
+        /* Get rings from nflib */
+        rx_ring = onvm_nflib_get_rx_ring(nf_info);
+        tx_ring = onvm_nflib_get_tx_ring(nf_info);
+        tx_stats = onvm_nflib_get_tx_stats(nf_info);
+
+        while (keep_running && rx_ring && tx_ring && tx_stats) {
+                tx_batch_size= 0;
+                /* Dequeue all packets in ring up to max possible. */
+		nb_pkts = rte_ring_dequeue_burst(rx_ring, pkts, PKT_READ_SIZE);
+
+                if(unlikely(nb_pkts == 0)) {
+                        continue;
+                }
+                /* Process all the packets */
+                for (i = 0; i < nb_pkts; i++) {
+                        meta = onvm_get_pkt_meta((struct rte_mbuf*)pkts[i]);
+                        packet_handler((struct rte_mbuf*)pkts[i], meta);
+                        pktsTX[tx_batch_size++] = pkts[i];
+                }
+
+                if (unlikely(tx_batch_size > 0 && rte_ring_enqueue_bulk(tx_ring, pktsTX, tx_batch_size) == -ENOBUFS)) {
+                        tx_stats->tx_drop[nf_info->instance_id] += tx_batch_size;
+                        for (j = 0; j < tx_batch_size; j++) {
+                                rte_pktmbuf_free(pktsTX[j]);
+                        }
+                } else {
+                        tx_stats->tx[nf_info->instance_id] += tx_batch_size;
+                }
+
+        }
+        onvm_nflib_stop();
+}
+
+
 int main(int argc, char *argv[]) {
         int arg_offset;
 
@@ -210,7 +278,11 @@ int main(int argc, char *argv[]) {
                 onvm_nflib_return_pkt(pkts[i]);
         }
 
-        onvm_nflib_run(nf_info, &packet_handler);
+        if (use_direct_rings) {
+                run_advanced_rings();
+        } else {
+                onvm_nflib_run(nf_info, &packet_handler);
+        }
         printf("If we reach here, program is ending");
         return 0;
 }

onvm/go.sh

@@ -2,6 +2,7 @@
 
 cpu=$1
 ports=$2
+virtaddr=$3
 
 SCRIPT=$(readlink -f "$0")
 SCRIPTPATH=$(dirname "$SCRIPT")
@@ -17,4 +18,9 @@ then
 fi
 
 sudo rm -rf /mnt/huge/rtemap_*
-sudo $SCRIPTPATH/onvm_mgr/onvm_mgr/$RTE_TARGET/onvm_mgr -l $cpu -n 4 --proc-type=primary  -- -p${ports}
+if [ -z $virtaddr ]
+then
+    sudo $SCRIPTPATH/onvm_mgr/onvm_mgr/$RTE_TARGET/onvm_mgr -l $cpu -n 4 --proc-type=primary  -- -p${ports}
+else
+    sudo $SCRIPTPATH/onvm_mgr/onvm_mgr/$RTE_TARGET/onvm_mgr -l $cpu -n 4 --proc-type=primary --base-virtaddr=$virtaddr  -- -p${ports}
+fi

onvm/onvm_nflib/onvm_nflib.c

@@ -70,6 +70,11 @@
 // Number of packets to attempt to read from queue
 #define PKT_READ_SIZE  ((uint16_t)32)
 
+// Possible NF packet consuming modes
+#define NF_MODE_UNKNOWN 0
+#define NF_MODE_SINGLE 1
+#define NF_MODE_RING 2
+
 
 /******************************Global Variables*******************************/
 
@@ -106,6 +111,10 @@ static uint16_t service_id = -1;
 static uint8_t keep_running = 1;
 
 
+// Mode this NF is running in (single packet, or direct ring manipulation)
+static uint8_t nf_mode = NF_MODE_UNKNOWN;
+
+
 // Shared data for default service chain
 static struct onvm_service_chain *default_chain;
 
@@ -281,6 +290,12 @@ onvm_nflib_run(
         void *pkts[PKT_READ_SIZE];
         struct onvm_pkt_meta* meta;
 
+        /* Don't allow conflicting NF modes */
+        if (nf_mode == NF_MODE_RING) {
+                return -1;
+        }
+        nf_mode = NF_MODE_SINGLE;
+
         printf("\nClient process %d handling packets\n", info->instance_id);
         printf("[Press Ctrl-C to quit ...]\n");
 
@@ -348,6 +363,45 @@ onvm_nflib_stop(void) {
 }
 
 
+struct rte_ring *
+onvm_nflib_get_tx_ring(__attribute__((__unused__)) struct onvm_nf_info* info) {
+        /* Don't allow conflicting NF modes */
+        if (nf_mode == NF_MODE_SINGLE) {
+                return NULL;
+        }
+
+        /* We should return the tx_ring associated with the info struct */
+        nf_mode = NF_MODE_RING;
+        return tx_ring;
+}
+
+
+struct rte_ring *
+onvm_nflib_get_rx_ring(__attribute__((__unused__)) struct onvm_nf_info* info) {
+        /* Don't allow conflicting NF modes */
+        if (nf_mode == NF_MODE_SINGLE) {
+                return NULL;
+        }
+
+        /* We should return the rx_ring associated with the info struct */
+        nf_mode = NF_MODE_RING;
+        return rx_ring;
+}
+
+
+volatile struct client_tx_stats *
+onvm_nflib_get_tx_stats(__attribute__((__unused__)) struct onvm_nf_info* info) {
+        /* Don't allow conflicting NF modes */
+        if (nf_mode == NF_MODE_SINGLE) {
+                return NULL;
+        }
+
+        /* We should return the tx_stats associated with the info struct */
+        nf_mode = NF_MODE_RING;
+        return tx_stats;
+}
+
+
 /******************************Helper functions*******************************/
 
 

onvm/onvm_nflib/onvm_nflib.h

@@ -7,6 +7,7 @@
  *   Copyright(c)
  *            2015-2016 George Washington University
  *            2015-2016 University of California Riverside
+ *            2016 Hewlett Packard Enterprise Development LP
  *   All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
@@ -108,8 +109,44 @@ onvm_nflib_return_pkt(struct rte_mbuf* pkt);
 
 /**
  * Stop this NF and clean up its memory
+ * Sends shutdown message to manager.
  */
 void
 onvm_nflib_stop(void);
 
+/**
+ * Return the tx_ring associated with this NF.
+ *
+ * @param info
+ *   an info struct describing this NF app.
+ * @return
+ *    pointer to tx_ring structure associated with info, NULL on error.
+ */
+struct rte_ring *
+onvm_nflib_get_tx_ring(struct onvm_nf_info* info);
+
+
+/**
+ * Return the rx_ring associated with this NF.
+ *
+ * @param info
+ *   an info struct describing this NF app.
+ * @return
+ *    pointer to rx_ring structure associated with info, NULL on error.
+ */
+struct rte_ring *
+onvm_nflib_get_rx_ring(struct onvm_nf_info* info);
+
+
+/**
+ * Return the tx_stats associated with this NF.
+ *
+ * @param info
+ *   an info struct describing this NF app.
+ * @return
+ *    pointer to tx_stats structure associated with info, NULL on error.
+ */
+volatile struct client_tx_stats *
+onvm_nflib_get_tx_stats(struct onvm_nf_info* info);
+
 #endif  // _ONVM_NFLIB_H_