Added L7 (nDPI) JSON collection

include/Flow.h

@@ -75,7 +75,7 @@ class Flow : public GenericHashEntry {
   FlowCollectionInfo *collection;
 
   /* Data collected from nProbe */
-
+  std::string l7_json;
   ICMPinfo *icmp_info;
   char *category_list_name_shared_pointer; /* NOTE: this is a pointer handled by
 					      Ntop::getPersistentCustomListNameById()
@@ -590,6 +590,8 @@ class Flow : public GenericHashEntry {
   void timeval_diff(struct timeval *begin, const struct timeval *end,
                     struct timeval *result, u_short divide_by_two);
   std::string getFlowInfo(bool isLuaRequest);
+  inline std::string getL7JSON() { return(l7_json); }
+  inline void setL7JSON(std::string j) { l7_json = j; }
   inline char *getFlowServerInfo() {
     return (isTLS() && protos.tls.client_requested_server_name)
                ? protos.tls.client_requested_server_name

include/ParsedFlow.h

@@ -38,6 +38,7 @@ class ParsedFlow : public ParsedFlowCore, public ParsedeBPF {
   char *ja4c_hash;
   char *flow_risk_info;
   char *external_alert;
+  char *l7_json;
   char *smtp_rcp_to, *smtp_mail_from;
   u_int32_t src_ip_addr_pre_nat, dst_ip_addr_pre_nat,
               src_ip_addr_post_nat, dst_ip_addr_post_nat;
@@ -109,6 +110,7 @@ class ParsedFlow : public ParsedFlowCore, public ParsedeBPF {
   inline void setJA4cHash(const char *str) { if(ja4c_hash != NULL) free(ja4c_hash);  if(str) { ja4c_hash = strdup(str); } else ja4c_hash = NULL; }
   inline void setRiskInfo(const char *str) { if(flow_risk_info != NULL) free(flow_risk_info);  if(str) { flow_risk_info = strdup(str); } else flow_risk_info = NULL; }
   inline void setExternalAlert(const char *str) { if(external_alert != NULL) free(external_alert);  if(str) { external_alert = strdup(str);} else external_alert = NULL; }
+  inline void setL7JSON(const char *str) { if(l7_json != NULL) free(l7_json);  if(str) { l7_json = strdup(str);} else l7_json = NULL; }
   inline void setTLSUnsafeCipher(u_int8_t v) { tls_unsafe_cipher = v; }
   inline void setTLSCipher(u_int16_t v) { tls_cipher = v; }
   inline void setFlowVerdict(u_int8_t v) { flow_verdict = v; }
@@ -149,6 +151,7 @@ class ParsedFlow : public ParsedFlowCore, public ParsedeBPF {
   inline char* getJA4cHash(bool setToNULL = false) { char *r = ja4c_hash; if(setToNULL) ja4c_hash = NULL; return(r); }
   inline char* getRiskInfo(bool setToNULL = false) { char *r = flow_risk_info; if(setToNULL) flow_risk_info  = NULL; return(r); }
   inline char* getExternalAlert(bool setToNULL = false) { char *r = external_alert; if(setToNULL) external_alert = NULL; return(r); }
+  inline char* getL7JSON(bool setToNULL = false) { char *r = l7_json; if(setToNULL) l7_json = NULL; return(r); }
   inline char* getEndReason(bool setToNull = false) { char *r = end_reason; if(setToNull) end_reason = NULL; return(r); }
   inline char* getSMTPRcptTo(bool setToNull = false) { char *r = smtp_rcp_to; if(setToNull) smtp_rcp_to = NULL; return(r); }
   inline char* getSMTPMailFrom(bool setToNull = false) { char *r = smtp_mail_from; if(setToNull) smtp_mail_from = NULL; return(r); }

include/ntop_flow.h

@@ -199,6 +199,7 @@
 #define L7_PROTO_NAME NTOP_BASE_ID + 119
 #define L7_INFO NTOP_BASE_ID + 539
 #define L7_CONFIDENCE NTOP_BASE_ID + 560
+#define L7_DETAILS NTOP_BASE_ID+593 /* nDPI JSON */
 #define DOWNSTREAM_TUNNEL_ID NTOP_BASE_ID + 120
 #define FLOW_USER_NAME NTOP_BASE_ID + 121
 #define NPROBE_INSTANCE_NAME NTOP_BASE_ID + 122

scripts/locales/en.lua

@@ -3275,6 +3275,7 @@ local lang = {
     ["looks_like_idle_flow_message"] = "This looks like an <font color=red>idle flow</font> with periodic transmissions just to keep it alive.",
     ["lost_packets"] = "Lost Packets",
     ["low_goodput"] = "Low Goodput",
+    ["l7_json"] = "nDPI Details",
     ["max_estimated_tcp_throughput"] = "Max (Estimated) TCP Throughput",
     ["max_packet_interarrival_time"] = "Max Packet Interarrival Time",
     ["mean_rtt"] = "Mean RTT",

scripts/lua/flow_details.lua

@@ -1584,6 +1584,11 @@ else
       end
    end
 
+   if(flow.l7_json ~= nil) then
+      print("<tr><th width=10%>" .. i18n("flow_details.l7_json") .. "</th><td colspan=2 style='overflow-wrap: anywhere !important; max-width: 200px; white-space: pre-wrap; word-break: keep-all; font-family: \"courier new\", courier, monospace;font-size: 13px;'>" .. flow.l7_json .. "</td></tr>\n")
+
+   end
+
    -- ######################################
 
    if (flow.entropy and flow.entropy.client and flow.entropy.server) then
@@ -1983,8 +1988,7 @@ else
       end
 
       if (snmpdevice ~= nil) then
-         local exporter_info_url = ntop.getHttpPrefix() .. "/lua/pro/enterprise/flowdevice_details.lua?ip=" ..
-             snmpdevice
+         local exporter_info_url = ntop.getHttpPrefix() .. "/lua/pro/enterprise/flowdevice_details.lua?ip=" .. snmpdevice
          print("<tr><th>" .. i18n("details.flow_exporter") .. "</th>")
          print("<td colspan=\"2\">" .. "<a href=" .. exporter_info_url .. ">" .. snmpdevice .. "</a></td></tr>")
 
@@ -2090,8 +2094,7 @@ else
          payload = string.reverse(string.sub(payload, idx))
       end
 
-      print(
-         "<tr><th width=10%>Payload</th><td colspan=2 style='overflow-wrap: anywhere !important; max-width: 200px; white-space: pre-wrap; word-break: keep-all; font-family: \"courier new\", courier, monospace;font-size: 13px;'>" ..
+      print("<tr><th width=10%>Payload</th><td colspan=2 style='overflow-wrap: anywhere !important; max-width: 200px; white-space: pre-wrap; word-break: keep-all; font-family: \"courier new\", courier, monospace;font-size: 13px;'>" ..
          payload .. "</td></tr>\n")
    end
 

src/Flow.cpp

@@ -3022,6 +3022,9 @@ void Flow::lua(lua_State *vm, AddressTree *ptree,
                                   protos.dns.last_return_code);
     }
 
+    if(l7_json.length() > 0)
+      lua_push_str_table_entry(vm, "l7_json", l7_json.c_str());
+
 #ifdef HAVE_NEDGE
     lua_push_uint64_table_entry(vm, "marker", marker);
 
@@ -5542,7 +5545,6 @@ void Flow::timeval_diff(struct timeval *begin, const struct timeval *end,
 
 /* *************************************** */
 
-/* FIXX this function is using buf only in a few cases */
 std::string Flow::getFlowInfo(bool isLuaRequest) {
   std::string info_field = std::string("");
 

src/ParsedFlow.cpp

@@ -55,7 +55,7 @@ ParsedFlow::ParsedFlow() : ParsedFlowCore(), ParsedeBPF() {
   memset(&custom_app, 0, sizeof(custom_app));
   wlan_ssid = NULL;
   memset(&wtp_mac_address, 0, sizeof(wtp_mac_address));
-
+  l7_json = NULL;
   has_parsed_ebpf = false;
 }
 
@@ -157,6 +157,11 @@ ParsedFlow::ParsedFlow(const ParsedFlow &pf) : ParsedFlowCore(pf), ParsedeBPF(pf
   else
     wlan_ssid = NULL;
 
+  if(pf.l7_json)
+    l7_json = strdup(pf.l7_json);
+  else
+    l7_json = NULL;
+
   memcpy(&wtp_mac_address, &pf.wtp_mac_address, sizeof(wtp_mac_address));
 
   tls_cipher = pf.tls_cipher;
@@ -233,6 +238,9 @@ void ParsedFlow::fromLua(lua_State *L, int index) {
           last_switched = Utils::str2epoch(lua_tostring(L, -1));
         } else if (!strcmp(key, "l4_proto")) {
           l4_proto = Utils::l4name2proto(lua_tostring(L, -1));
+        } else if (!strcmp(key, "l7_json")) {
+          if (l7_json) free(l7_json);
+          l7_json = strdup(lua_tostring(L, -1));
         } else {
           addAdditionalField(key, json_object_new_string(lua_tostring(L, -1)));
           ntop->getTrace()->traceEvent(TRACE_DEBUG,
@@ -336,6 +344,7 @@ void ParsedFlow::freeMemory() {
   if (dhcp_client_name)     { free(dhcp_client_name); dhcp_client_name = NULL; }
   if (sip_call_id)          { free(sip_call_id); sip_call_id = NULL; }
   if (wlan_ssid)            { free(wlan_ssid); wlan_ssid = NULL; }
+  if (l7_json)              { free(l7_json); l7_json = NULL; }
 }
 
 /* *************************************** */

src/ParserInterface.cpp

@@ -399,6 +399,9 @@ bool ParserInterface::processFlow(ParsedFlow *zflow) {
   if (zflow->getWLANSSID())
     flow->setWLANInfo(zflow->getWLANSSID(), zflow->getWTPMACAddress());
 
+  if (zflow->getL7JSON())
+    flow->setL7JSON(zflow->getL7JSON());
+
   if (zflow->getSIPCallId()) flow->setSIPCallId(zflow->getSIPCallId());
 
   if (zflow->isSwapped()) flow->set_swap_done();

src/ZMQParserInterface.cpp

@@ -1050,6 +1050,9 @@ bool ZMQParserInterface::parsePENZeroField(ParsedFlow *const flow,
   case WTP_MAC_ADDRESS:
     if (value->string) flow->setWTPMACAddress(value->string);
     break;
+  case L7_DETAILS:
+    if (value->string) flow->setL7JSON(value->string);
+    break;
   default:
     ntop->getTrace()->traceEvent(TRACE_INFO,
 				 "Skipping no-PEN flow fieldId %u", field);