Skip to content

Commit

Permalink
Preliminary work to rework struct ndpi_flow_struct (#2705)
Browse files Browse the repository at this point in the history 
No significant changes:
 * Move around some fields to avoid holes in the structures.
 * Some fields are about protocols based only on TCP.
 * Remove some unused (or set but never read) fields.

See #2631
  • Loading branch information
@IvanNardi
IvanNardi authored Feb 4, 2025
1 parent 642cf57 commit 07eb92f
Show file tree
Hide file tree
Showing 6 changed files with 124 additions and 129 deletions.
174 changes: 86 additions & 88 deletions src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,51 @@ struct ndpi_lru_cache {
/* ************************************************** */

struct ndpi_flow_tcp_struct {
/* TCP sequence number */
u_int32_t next_tcp_seq_nr[2];
u_int16_t last_tcp_pkt_payload_len;

/* Part of the TCP header */
u_int8_t cli2srv_tcp_flags, srv2cli_tcp_flags;
u_int32_t seen_syn:1, seen_syn_ack:1, seen_ack:1;

/* NDPI_PROTOCOL_IRC */
u_int32_t irc_3a_counter:3;

/* NDPI_PROTOCOL_USENET */
u_int32_t usenet_stage:2;

/* NDPI_PROTOCOL_HTTP */
u_int32_t http_stage:3;
u_int32_t http_asymmetric_stage:2;

/* NDPI_PROTOCOL_GNUTELLA */
u_int32_t gnutella_stage:2; // 0 - 2

/* NDPI_PROTOCOL_SSH */
u_int32_t ssh_stage:3;

/* NDPI_PROTOCOL_VNC */
u_int32_t vnc_stage:2; // 0 - 3

/* NDPI_PROTOCOL_TELNET */
u_int32_t telnet_stage:2; // 0 - 2

/* NDPI_PROTOCOL_RADMIN */
u_int32_t radmin_stage:1;

/* NDPI_PROTOCOL_FTP_CONTROL */
u_int32_t ftp_control_stage:2;

/* NDPI_PROTOCOL_SOAP */
u_int32_t soap_stage:1;

/* NDPI_PROTOCOL_SOCKS */
u_int32_t socks5_stage:2, socks4_stage:2;

/* NDPI_PROTOCOL_Z3950 */
u_int32_t z3950_stage:2;

/* NDPI_PROTOCOL_MAIL_SMTP */
/* NDPI_PROTOCOL_MAIL_POP */
/* NDPI_PROTOCOL_MAIL_IMAP */
Expand All @@ -815,6 +860,9 @@ struct ndpi_flow_tcp_struct {
char username[32], password[16];
} ftp_imap_pop_smtp;

/* NDPI_PROTOCOL_LOTUS_NOTES */
u_int8_t lotus_notes_packet_id;

/* NDPI_PROTOCOL_MAIL_SMTP */
u_int16_t smtp_command_bitmask;

Expand All @@ -830,31 +878,11 @@ struct ndpi_flow_tcp_struct {
/* NDPI_PROTOCOL_GNUTELLA */
u_int8_t gnutella_msg_id[3];

/* NDPI_PROTOCOL_IRC */
u_int32_t irc_3a_counter:3;

/* NDPI_PROTOCOL_USENET */
u_int32_t usenet_stage:2;

/* NDPI_PROTOCOL_HTTP */
u_int32_t http_stage:3;
u_int32_t http_asymmetric_stage:2;

/* NDPI_PROTOCOL_GNUTELLA */
u_int32_t gnutella_stage:2; // 0 - 2

/* NDPI_PROTOCOL_SSH */
u_int32_t ssh_stage:3;

/* NDPI_PROTOCOL_VNC */
u_int32_t vnc_stage:2; // 0 - 3

/* NDPI_PROTOCOL_TELNET */
u_int32_t telnet_stage:2; // 0 - 2
/* NDPI_PROTOCOL_NEST_LOG_SINK */
u_int8_t nest_log_sink_matches;

/* NDPI_PROTOCOL_RTMP */
u_int32_t rtmp_stage:2;
u_int16_t rtmp_client_buffer_len;
/* NDPI_PROTOCOL_MEMCACHED */
u_int8_t memcached_matches;

struct {
/* NDPI_PROTOCOL_TLS */
Expand All @@ -863,12 +891,16 @@ struct ndpi_flow_tcp_struct {
int16_t tls_application_blocks_len[NDPI_MAX_NUM_TLS_APPL_BLOCKS]; /* + = src->dst, - = dst->src */
} tls;

/* NDPI_PROTOCOL_ZMQ */
u_char prev_zmq_pkt[10];
u_int8_t prev_zmq_pkt_len;

/* NDPI_PROTOCOL_RTMP */
u_int16_t rtmp_client_buffer_len;
u_int32_t rtmp_stage:2;

/* NDPI_PROTOCOL_POSTGRES */
u_int32_t postgres_stage:3;

/* Part of the TCP header. */
u_int32_t seen_syn:1, seen_syn_ack:1, seen_ack:1, __notused:29;
u_int8_t cli2srv_tcp_flags, srv2cli_tcp_flags;

/* NDPI_PROTOCOL_ICECAST */
u_int32_t icecast_stage:1;
Expand All @@ -884,25 +916,6 @@ struct ndpi_flow_tcp_struct {

/* NDPI_PROTOCOL_MAIL_IMAP */
u_int32_t mail_imap_stage:3, mail_imap_starttls:2;

/* NDPI_PROTOCOL_SOAP */
u_int32_t soap_stage:1;

/* NDPI_PROTOCOL_LOTUS_NOTES */
u_int8_t lotus_notes_packet_id;

/* NDPI_PROTOCOL_ZMQ */
u_int8_t prev_zmq_pkt_len;
u_char prev_zmq_pkt[10];

/* NDPI_PROTOCOL_MEMCACHED */
u_int8_t memcached_matches;

/* NDPI_PROTOCOL_NEST_LOG_SINK */
u_int8_t nest_log_sink_matches;

/* NDPI_PROTOCOL_RADMIN */
u_int32_t radmin_stage:1;
};

/* ************************************************** */
Expand All @@ -928,27 +941,33 @@ struct ndpi_flow_udp_struct {
/* NDPI_PROTOCOL_ZOOM */
u_int32_t zoom_p2p:1;

/* NDPI_PROTOCOL_EPICGAMES */
u_int32_t epicgames_stage:1;
u_int32_t epicgames_word;

/* NDPI_PROTOCOL_RAKNET */
u_int32_t raknet_custom:1;

/* NDPI_PROTOCOL_EAQ */
u_int8_t eaq_pkt_id;
u_int32_t eaq_sequence;
/* NDPI_PROTOCOL_MUMBLE */
u_int32_t mumble_stage:1;

/* NDPI_PROTOCOL_EPICGAMES */
u_int32_t epicgames_stage:1;
u_int32_t epicgames_word;

/* NDPI_PROTOCOL_RX */
u_int32_t rx_conn_epoch;
u_int32_t rx_conn_id;

/* NDPI_PROTOCOL_WIREGUARD */
u_int32_t wireguard_peer_index[2];
u_int8_t wireguard_stage;

/* NDPI_PROTOCOL_MEMCACHED */
u_int8_t memcached_matches;

/* NDPI_PROTOCOL_WIREGUARD */
u_int8_t wireguard_stage;
u_int32_t wireguard_peer_index[2];
/* NDPI_PROTOCOL_EAQ */
u_int8_t eaq_pkt_id;
u_int32_t eaq_sequence;

/* NDPI_PROTOCOL_MUMBLE */
u_int64_t mumble_ident;

/* NDPI_PROTOCOL_QUIC */
u_int8_t *quic_reasm_buf;
Expand All @@ -975,9 +994,6 @@ struct ndpi_flow_udp_struct {
u_int16_t tftp_data_num;
u_int16_t tftp_ack_num;

/* NDPI_PROTOCOL_MUMBLE */
u_int8_t mumble_stage:1;
u_int64_t mumble_ident;
};

/* ************************************************** */
Expand Down Expand Up @@ -1272,7 +1288,7 @@ struct ndpi_flow_struct {
u_int16_t guessed_protocol_id; /* Classification by-port. Set with the first pkt and never updated */
u_int16_t guessed_protocol_id_by_ip; /* Classification by-ip. Set with the first pkt and never updated */
u_int16_t fast_callback_protocol_id; /* Partial/incomplete classification. Used internally as first callback when iterating all the protocols */
u_int16_t guessed_category, guessed_header_category;
u_int16_t guessed_header_category;
u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;
u_int8_t monitoring:1, _pad:7;
Expand All @@ -1282,13 +1298,6 @@ struct ndpi_flow_struct {

/* First Packet Classification info */
struct ndpi_fpc_info fpc;

/*
if ndpi_struct->direction_detect_disable == 1
tcp sequence number connection tracking
*/
u_int32_t next_tcp_seq_nr[2];
u_int16_t last_tcp_pkt_payload_len;

/* Flow addresses (useful for LRU lookups in ndpi_detection_giveup())
and ports. All in *network* byte order.
Expand All @@ -1310,6 +1319,15 @@ struct ndpi_flow_struct {

u_int64_t last_packet_time_ms;

ndpi_protocol_category_t category;

/* Counters with only packets with L5 data (ie no TCP SYN, pure ACKs, ...) */
u_int16_t packet_counter;
u_int16_t packet_direction_counter[2];
/* Counters with all packets even those without payload */
u_int16_t all_packets_counter;
u_int16_t packet_direction_complete_counter[2];

/*
the tcp / udp / other l4 value union
used to reduce the number of bytes for tcp or udp protocol states
Expand Down Expand Up @@ -1548,17 +1566,6 @@ struct ndpi_flow_struct {
/* protocols which have marked a connection as this connection cannot be protocol XXX, multiple u_int64_t */
NDPI_PROTOCOL_BITMASK excluded_protocol_bitmask;

ndpi_protocol_category_t category;

/* Only packets with L5 data (ie no TCP SYN, pure ACKs, ...) */
u_int16_t packet_counter; // can be 0 - 65000
u_int16_t packet_direction_counter[2];
u_int8_t packet_direction_with_payload_observed[2]; /* 0 = no packet with payload observed, 1 = at least one packet with payload observed */

/* All packets even those without payload */
u_int16_t all_packets_counter;
u_int16_t packet_direction_complete_counter[2]; // can be 0 - 65000

/* NDPI_PROTOCOL_BITTORRENT */
u_int8_t bittorrent_stage; // can be 0 - 255
u_int8_t bt_check_performed : 1;
Expand All @@ -1569,18 +1576,9 @@ struct ndpi_flow_struct {
/* NDPI_PROTOCOL_ZATTOO */
u_int8_t zattoo_stage:3;

/* NDPI_PROTOCOL_SOCKS */
u_int8_t socks5_stage:2, socks4_stage:2; // 0 - 3

/* NDPI_PROTOCOL_FTP_CONTROL */
u_int8_t ftp_control_stage:2;

/* NDPI_PROTOCOL_STARCRAFT */
u_int8_t starcraft_udp_stage : 3; // 0-7

/* NDPI_PROTOCOL_Z3950 */
u_int8_t z3950_stage : 2; // 0-3

/* NDPI_PROTOCOL_OOKLA */
u_int8_t ookla_stage : 1;

Expand Down
29 changes: 13 additions & 16 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7281,7 +7281,7 @@ static void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_s
if(ndpi_str->cfg.tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) {
NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n");
packet->tcp_retransmission = 1;
} else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
} else if(flow->l4.tcp.next_tcp_seq_nr[0] == 0 || flow->l4.tcp.next_tcp_seq_nr[1] == 0 ||
(tcph->syn && flow->packet_counter == 0)) {
/* initialize tcp sequence counters */
/* the ack flag needs to be set to get valid sequence numbers from the other
Expand All @@ -7294,41 +7294,41 @@ static void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_s
* If we receive multiple syn-ack (before any real data), keep the last one
*/
if(tcph->ack != 0) {
flow->next_tcp_seq_nr[packet->packet_direction] =
flow->l4.tcp.next_tcp_seq_nr[packet->packet_direction] =
ntohl(tcph->seq) + (tcph->syn ? 1 : packet->payload_packet_len);

/*
Check to avoid discrepancies in case we analyze a flow that does not start with SYN...
but that is already started when nDPI being to process it. See also (***) below
*/
if(flow->num_processed_pkts > 1)
flow->next_tcp_seq_nr[1 - packet->packet_direction] = ntohl(tcph->ack_seq);
flow->l4.tcp.next_tcp_seq_nr[1 - packet->packet_direction] = ntohl(tcph->ack_seq);
}
} else if(packet->payload_packet_len > 0) {
/* check tcp sequence counters */
if(((u_int32_t)(ntohl(tcph->seq) - flow->next_tcp_seq_nr[packet->packet_direction])) >
if(((u_int32_t)(ntohl(tcph->seq) - flow->l4.tcp.next_tcp_seq_nr[packet->packet_direction])) >
ndpi_str->tcp_max_retransmission_window_size) {
if(flow->last_tcp_pkt_payload_len > 0)
if(flow->l4.tcp.last_tcp_pkt_payload_len > 0)
packet->tcp_retransmission = 1;

/* CHECK IF PARTIAL RETRY IS HAPPENING */
if((flow->next_tcp_seq_nr[packet->packet_direction] - ntohl(tcph->seq) <
if((flow->l4.tcp.next_tcp_seq_nr[packet->packet_direction] - ntohl(tcph->seq) <
packet->payload_packet_len)) {
if(flow->num_processed_pkts > 1) /* See also (***) above */
flow->next_tcp_seq_nr[packet->packet_direction] = ntohl(tcph->seq) + packet->payload_packet_len;
flow->l4.tcp.next_tcp_seq_nr[packet->packet_direction] = ntohl(tcph->seq) + packet->payload_packet_len;
}
}
else {
flow->next_tcp_seq_nr[packet->packet_direction] = ntohl(tcph->seq) + packet->payload_packet_len;
flow->l4.tcp.next_tcp_seq_nr[packet->packet_direction] = ntohl(tcph->seq) + packet->payload_packet_len;
}
}

if(tcph->rst) {
flow->next_tcp_seq_nr[0] = 0;
flow->next_tcp_seq_nr[1] = 0;
flow->l4.tcp.next_tcp_seq_nr[0] = 0;
flow->l4.tcp.next_tcp_seq_nr[1] = 0;
}

flow->last_tcp_pkt_payload_len = packet->payload_packet_len;
flow->l4.tcp.last_tcp_pkt_payload_len = packet->payload_packet_len;
} else if(udph != NULL) {
if(ndpi_str->cfg.direction_detect_enabled &&
(udph->source != udph->dest))
Expand Down Expand Up @@ -7419,9 +7419,6 @@ static void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_s
flow->packet_direction_complete_counter[packet->packet_direction]++;
}

if(packet->payload_packet_len > 0)
flow->packet_direction_with_payload_observed[packet->packet_direction] = 1;

if(!ndpi_is_multi_or_broadcast(packet)) {
/* ! (multicast or broadcast) */

Expand Down Expand Up @@ -8002,8 +7999,8 @@ static void ndpi_check_probing_attempt(struct ndpi_detection_module_struct *ndpi
if((flow->l4_proto == IPPROTO_TCP)
&& (flow->l4.tcp.cli2srv_tcp_flags & TH_PUSH)
&& (flow->l4.tcp.srv2cli_tcp_flags & TH_PUSH)) {
if(flow->packet_direction_with_payload_observed[0]
&& flow->packet_direction_with_payload_observed[1]) {
if(flow->packet_direction_counter[0]
&& flow->packet_direction_counter[1]) {
/* Both directions observed */
/* Nothing to do */
} else {
Expand Down
Loading

0 comments on commit 07eb92f

Please sign in to comment.