Skip to content

Commit

Permalink
quic: add debug visitor to export various quic stats from quiche (env…
Browse files Browse the repository at this point in the history
…oyproxy#36813)

Signed-off-by: Greg Greenway <[email protected]>
  • Loading branch information
ggreenway authored Oct 29, 2024
1 parent d58be6e commit ec94989
Show file tree
Hide file tree
Showing 31 changed files with 792 additions and 68 deletions.
1 change: 1 addition & 0 deletions api/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@ proto_library(
"//envoy/extensions/outlier_detection_monitors/consecutive_errors/v3:pkg",
"//envoy/extensions/path/match/uri_template/v3:pkg",
"//envoy/extensions/path/rewrite/uri_template/v3:pkg",
"//envoy/extensions/quic/connection_debug_visitor/quic_stats/v3:pkg",
"//envoy/extensions/quic/connection_debug_visitor/v3:pkg",
"//envoy/extensions/quic/connection_id_generator/v3:pkg",
"//envoy/extensions/quic/crypto_stream/v3:pkg",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.

load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")

licenses(["notice"]) # Apache 2

api_proto_package(
deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
syntax = "proto3";

package envoy.extensions.quic.connection_debug_visitor.quic_stats.v3;

import "google/protobuf/duration.proto";

import "udpa/annotations/status.proto";
import "validate/validate.proto";

option java_package = "io.envoyproxy.envoy.extensions.quic.connection_debug_visitor.quic_stats.v3";
option java_outer_classname = "QuicStatsProto";
option java_multiple_files = true;
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/connection_debug_visitor/quic_stats/v3;quic_statsv3";
option (udpa.annotations.file_status).package_version_status = ACTIVE;

// [#protodoc-title: QUIC stats config]
// [#extension: envoy.quic.connection_debug_visitor.quic_stats]

// Configuration for a QUIC debug visitor which emits stats from the underlying QUIC transport.
message Config {
// Period to update stats while the connection is open. If unset, updates only happen when the
// connection is closed. Stats are always updated one final time when the connection is closed.
google.protobuf.Duration update_period = 2 [(validate.rules).duration = {gte {nanos: 1000000}}];
}
1 change: 1 addition & 0 deletions api/versioning/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ proto_library(
"//envoy/extensions/outlier_detection_monitors/consecutive_errors/v3:pkg",
"//envoy/extensions/path/match/uri_template/v3:pkg",
"//envoy/extensions/path/rewrite/uri_template/v3:pkg",
"//envoy/extensions/quic/connection_debug_visitor/quic_stats/v3:pkg",
"//envoy/extensions/quic/connection_debug_visitor/v3:pkg",
"//envoy/extensions/quic/connection_id_generator/v3:pkg",
"//envoy/extensions/quic/crypto_stream/v3:pkg",
Expand Down
4 changes: 4 additions & 0 deletions changelogs/current.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ new_features:
change: |
Added %DOWNSTREAM_LOCAL_EMAIL_SAN%, %DOWNSTREAM_PEER_EMAIL_SAN%, %DOWNSTREAM_LOCAL_OTHERNAME_SAN% and
%DOWNSTREAM_PEER_OTHERNAME_SAN% substitution formatters.
- area: quic
change: |
Added :ref:`QUIC stats debug visitor <envoy_v3_api_msg_extensions.quic.connection_debug_visitor.quic_stats.v3.Config>` to
get more stats from the QUIC transport.
- area: http_inspector
change: |
Added default-false ``envoy.reloadable_features.http_inspector_use_balsa_parser`` for HttpInspector to use BalsaParser.
Expand Down
1 change: 1 addition & 0 deletions docs/root/api-v3/config/quic/quic_extensions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ Quic extensions
../../extensions/quic/connection_id_generator/v3/*
../../extensions/quic/server_preferred_address/v3/*
../../extensions/quic/connection_debug_visitor/v3/*
../../extensions/quic/connection_debug_visitor/quic_stats/v3/*
24 changes: 24 additions & 0 deletions docs/root/configuration/listeners/stats.rst
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,30 @@ The following UDP statistics are available for UDP listeners and are rooted at

downstream_rx_datagram_dropped, Counter, Number of datagrams dropped due to kernel overflow or truncation

.. _config_listener_stats_quic:

QUIC statistics
---------------

The following QUIC statistics, which are available when using the :ref:`QUIC stats debug visitor <envoy_v3_api_msg_extensions.quic.connection_debug_visitor.quic_stats.v3.Config>`,
are rooted at *listener.<address>.quic_stats.*:

.. csv-table::
:header: Name, Type, Description
:widths: 1, 1, 2

cx_tx_packets_total, Counter, Total packets transmitted
cx_tx_packets_retransmitted_total, Counter, Total packets retransmitted
cx_tx_amplification_throttling_total, Counter, Total number of packets throttled during the server handshake response. This often indicates that the TLS certificate chain is too long to be transmitted without an additional network round trip.
cx_rx_packets_total, Counter, Total number of packets received
cx_path_degrading_total, Counter, Number of times that network path degradation was detected
cx_forward_progress_after_path_degrading_total, Counter, Number of times that forward progress was made after the path degraded
cx_rtt_us, Histogram, Smoothed round trip time estimate in microseconds
cx_tx_estimated_bandwidth, Histogram, Estimated connection bandwith in bytes per second
cx_tx_percent_retransmitted_packets, Histogram, Percent of packets on a connection which were retransmistted
cx_tx_mtu, Histogram, The maximum packet size that will be sent for a connection
cx_rx_mtu, Histogram, The size of the largest packet received from the peer

.. _config_listener_stats_per_handler:

Per-handler Listener Stats
Expand Down
2 changes: 1 addition & 1 deletion source/common/listener_manager/listener_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,7 @@ ListenerImpl::buildUdpListenerFactory(const envoy::config::listener::v3::Listene
}
udp_listener_config_->listener_factory_ = std::make_unique<Quic::ActiveQuicListenerFactory>(
config.udp_listener_config().quic_options(), concurrency, quic_stat_names_,
validation_visitor_, listener_factory_context_->serverFactoryContext());
validation_visitor_, *listener_factory_context_);
#if UDP_GSO_BATCH_WRITER_COMPILETIME_SUPPORT
// TODO(mattklein123): We should be able to use GSO without QUICHE/QUIC. Right now this causes
// non-QUIC integration tests to fail, which I haven't investigated yet. Additionally, from
Expand Down
2 changes: 1 addition & 1 deletion source/common/quic/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ envoy_cc_library(
"//envoy/common:optref_lib",
"//envoy/common:pure_lib",
"//envoy/config:typed_config_interface",
"//envoy/server:process_context_interface",
"//envoy/server:factory_context_interface",
"//envoy/stream_info:stream_info_interface",
"@com_github_google_quiche//:quic_core_connection_lib",
"@com_github_google_quiche//:quic_core_session_lib",
Expand Down
19 changes: 10 additions & 9 deletions source/common/quic/active_quic_listener.cc
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ ActiveQuicListener::ActiveQuicListener(
crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper),
std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, *config_, stats_,
per_worker_stats_, dispatcher, listen_socket_, quic_stat_names, crypto_server_stream_factory_,
*connection_id_generator_, std::move(debug_visitor_factory));
*connection_id_generator_, debug_visitor_factory);

// Create udp_packet_writer
Network::UdpPacketWriterPtr udp_packet_writer =
Expand Down Expand Up @@ -261,7 +261,7 @@ void ActiveQuicListener::closeConnectionsWithFilterChain(const Network::FilterCh
ActiveQuicListenerFactory::ActiveQuicListenerFactory(
const envoy::config::listener::v3::QuicProtocolOptions& config, uint32_t concurrency,
QuicStatNames& quic_stat_names, ProtobufMessage::ValidationVisitor& validation_visitor,
Server::Configuration::ServerFactoryContext& context)
Server::Configuration::ListenerFactoryContext& context)
: concurrency_(concurrency), enabled_(config.enabled()), quic_stat_names_(quic_stat_names),
packets_to_read_to_connection_count_ratio_(
PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, packets_to_read_to_connection_count_ratio,
Expand Down Expand Up @@ -313,12 +313,13 @@ ActiveQuicListenerFactory::ActiveQuicListenerFactory(

// Initialize connection debug visitor factory if one is configured.
if (config.has_connection_debug_visitor_config()) {
connection_debug_visitor_factory_ =
Config::Utility::getAndCheckFactory<EnvoyQuicConnectionDebugVisitorFactoryInterface>(
auto& factory =
Config::Utility::getAndCheckFactory<EnvoyQuicConnectionDebugVisitorFactoryFactoryInterface>(
config.connection_debug_visitor_config());
if (connection_debug_visitor_factory_.has_value()) {
connection_debug_visitor_factory_->setContext(context_.processContext());
}
ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig(
config.connection_debug_visitor_config().typed_config(),
context_.messageValidationVisitor(), factory);
connection_debug_visitor_factory_ = factory.createFactory(*message, context_);
}

// Initialize connection ID generator factory.
Expand Down Expand Up @@ -349,7 +350,7 @@ ActiveQuicListenerFactory::ActiveQuicListenerFactory(
*Config::Utility::translateToFactoryConfig(config.server_preferred_address_config(),
validation_visitor,
server_preferred_address_config_factory),
validation_visitor, context_);
validation_visitor, context_.serverFactoryContext());
}

worker_selector_ =
Expand Down Expand Up @@ -437,7 +438,7 @@ ActiveQuicListenerFactory::createActiveQuicListener(
listener_config, quic_config, kernel_worker_routing, enabled, quic_stat_names,
packets_to_read_to_connection_count_ratio, receive_ecn_, crypto_server_stream_factory,
proof_source_factory, std::move(cid_generator), worker_selector_,
connection_debug_visitor_factory_, reject_new_connections_);
makeOptRefFromPtr(connection_debug_visitor_factory_.get()), reject_new_connections_);
}

} // namespace Quic
Expand Down
6 changes: 3 additions & 3 deletions source/common/quic/active_quic_listener.h
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
ActiveQuicListenerFactory(const envoy::config::listener::v3::QuicProtocolOptions& config,
uint32_t concurrency, QuicStatNames& quic_stat_names,
ProtobufMessage::ValidationVisitor& validation_visitor,
Server::Configuration::ServerFactoryContext& context);
Server::Configuration::ListenerFactoryContext& context);

// Network::ActiveUdpListenerFactory.
Network::ConnectionHandler::ActiveUdpListenerPtr
Expand Down Expand Up @@ -147,7 +147,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
crypto_server_stream_factory_;
absl::optional<std::reference_wrapper<EnvoyQuicProofSourceFactoryInterface>>
proof_source_factory_;
EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef connection_debug_visitor_factory_;
EnvoyQuicConnectionDebugVisitorFactoryInterfacePtr connection_debug_visitor_factory_;
EnvoyQuicConnectionIdGeneratorFactoryPtr quic_cid_generator_factory_;
EnvoyQuicServerPreferredAddressConfigPtr server_preferred_address_config_;
quic::QuicConfig quic_config_;
Expand All @@ -159,7 +159,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
const Network::Socket::OptionsSharedPtr options_{std::make_shared<Network::Socket::Options>()};
QuicConnectionIdWorkerSelector worker_selector_;
bool kernel_worker_routing_{};
Server::Configuration::ServerFactoryContext& context_;
Server::Configuration::ListenerFactoryContext& context_;
bool reject_new_connections_{};

static bool disable_kernel_bpf_packet_routing_for_test_;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
#include <memory>
#include <string>

#include "envoy/common/optref.h"
#include "envoy/common/pure.h"
#include "envoy/config/typed_config.h"
#include "envoy/server/process_context.h"
#include "envoy/server/factory_context.h"
#include "envoy/stream_info/stream_info.h"

#include "quiche/quic/core/quic_connection.h"
Expand All @@ -15,23 +14,29 @@
namespace Envoy {
namespace Quic {

class EnvoyQuicConnectionDebugVisitorFactoryInterface : public Config::TypedFactory {
class EnvoyQuicConnectionDebugVisitorFactoryInterface {
public:
std::string category() const override { return "envoy.quic.connection_debug_visitor"; }

void setContext(Envoy::ProcessContextOptRef context) { context_ = context; }
virtual ~EnvoyQuicConnectionDebugVisitorFactoryInterface() = default;

// Returns a debug visitor to be attached to a Quic Connection.
virtual std::unique_ptr<quic::QuicConnectionDebugVisitor>
createQuicConnectionDebugVisitor(quic::QuicSession* session,
createQuicConnectionDebugVisitor(Event::Dispatcher& dispatcher, quic::QuicSession& session,
const StreamInfo::StreamInfo& stream_info) PURE;

protected:
Envoy::ProcessContextOptRef context_;
};

using EnvoyQuicConnectionDebugVisitorFactoryInterfacePtr =
std::unique_ptr<EnvoyQuicConnectionDebugVisitorFactoryInterface>;
using EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef =
OptRef<EnvoyQuicConnectionDebugVisitorFactoryInterface>;

class EnvoyQuicConnectionDebugVisitorFactoryFactoryInterface : public Config::TypedFactory {
public:
std::string category() const override { return "envoy.quic.connection_debug_visitor"; }

virtual EnvoyQuicConnectionDebugVisitorFactoryInterfacePtr
createFactory(const Protobuf::Message& config,
Server::Configuration::ListenerFactoryContext& listener_context) PURE;
};

} // namespace Quic
} // namespace Envoy
4 changes: 2 additions & 2 deletions source/common/quic/envoy_quic_dispatcher.cc
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ EnvoyQuicDispatcher::EnvoyQuicDispatcher(
Network::Socket& listen_socket, QuicStatNames& quic_stat_names,
EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory,
quic::ConnectionIdGeneratorInterface& generator,
EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef&& debug_visitor_factory)
EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory)
: quic::QuicDispatcher(&quic_config, crypto_config, version_manager, std::move(helper),
std::make_unique<EnvoyQuicCryptoServerStreamHelper>(),
std::move(alarm_factory), expected_server_connection_id_length,
Expand All @@ -65,7 +65,7 @@ EnvoyQuicDispatcher::EnvoyQuicDispatcher(
quic_stats_(generateStats(listener_config.listenerScope())),
connection_stats_({QUIC_CONNECTION_STATS(
POOL_COUNTER_PREFIX(listener_config.listenerScope(), "quic.connection"))}),
debug_visitor_factory_(std::move(debug_visitor_factory)) {}
debug_visitor_factory_(debug_visitor_factory) {}

void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_id,
quic::QuicErrorCode error,
Expand Down
2 changes: 1 addition & 1 deletion source/common/quic/envoy_quic_dispatcher.h
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher {
Network::Socket& listen_socket, QuicStatNames& quic_stat_names,
EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory,
quic::ConnectionIdGeneratorInterface& generator,
EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef&& debug_visitor_factory);
EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory);

// quic::QuicDispatcher
void OnConnectionClosed(quic::QuicConnectionId connection_id, quic::QuicErrorCode error,
Expand Down
3 changes: 2 additions & 1 deletion source/common/quic/envoy_quic_server_session.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ EnvoyQuicServerSession::EnvoyQuicServerSession(
#endif
// If a factory is available, create a debug visitor and attach it to the connection.
if (debug_visitor_factory.has_value()) {
debug_visitor_ = debug_visitor_factory->createQuicConnectionDebugVisitor(this, streamInfo());
debug_visitor_ =
debug_visitor_factory->createQuicConnectionDebugVisitor(dispatcher, *this, streamInfo());
quic_connection_->set_debug_visitor(debug_visitor_.get());
}
quic_connection_->set_context_listener(
Expand Down
3 changes: 2 additions & 1 deletion source/extensions/extensions_build_config.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,8 @@ EXTENSIONS = {
"envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source",
"envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config",
"envoy.quic.server_preferred_address.datasource": "//source/extensions/quic/server_preferred_address:datasource_server_preferred_address_config_factory_config",
"envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor:envoy_quic_connection_debug_visitor_basic",
"envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor/basic:envoy_quic_connection_debug_visitor_basic",
"envoy.quic.connection_debug_visitor.quic_stats": "//source/extensions/quic/connection_debug_visitor/quic_stats:config",

#
# UDP packet writers
Expand Down
7 changes: 7 additions & 0 deletions source/extensions/extensions_metadata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1813,6 +1813,13 @@ envoy.quic.connection_debug_visitor.basic:
status: alpha
type_urls:
- envoy.extensions.quic.connection_debug_visitor.v3.BasicConfig
envoy.quic.connection_debug_visitor.quic_stats:
categories:
- envoy.quic.connection_debug_visitor
security_posture: robust_to_untrusted_downstream_and_upstream
status: alpha
type_urls:
- envoy.extensions.quic.connection_debug_visitor.quic_stats.v3.Config
envoy.filters.network.generic_proxy:
categories:
- envoy.filters.network
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#include "source/extensions/quic/connection_debug_visitor/envoy_quic_connection_debug_visitor_basic.h"
#include "source/extensions/quic/connection_debug_visitor/basic/envoy_quic_connection_debug_visitor_basic.h"

#include <memory>

Expand All @@ -17,18 +17,18 @@ namespace Quic {
void EnvoyQuicConnectionDebugVisitorBasic::OnConnectionClosed(
const quic::QuicConnectionCloseFrame& frame, quic::ConnectionCloseSource source) {
ENVOY_LOG(info, "Quic connection from {} with id {} closed {} with details: {}",
session_->peer_address().ToString(), session_->connection_id().ToString(),
session_.peer_address().ToString(), session_.connection_id().ToString(),
quic::ConnectionCloseSourceToString(source), frame.error_details);
}

std::unique_ptr<quic::QuicConnectionDebugVisitor>
EnvoyQuicConnectionDebugVisitorFactoryBasic::createQuicConnectionDebugVisitor(
quic::QuicSession* session, const StreamInfo::StreamInfo& stream_info) {
Event::Dispatcher&, quic::QuicSession& session, const StreamInfo::StreamInfo& stream_info) {
return std::make_unique<EnvoyQuicConnectionDebugVisitorBasic>(session, stream_info);
}

REGISTER_FACTORY(EnvoyQuicConnectionDebugVisitorFactoryBasic,
EnvoyQuicConnectionDebugVisitorFactoryInterface);
REGISTER_FACTORY(EnvoyQuicConnectionDebugVisitorFactoryFactoryBasic,
EnvoyQuicConnectionDebugVisitorFactoryFactoryInterface);

} // namespace Quic
} // namespace Envoy
Loading

0 comments on commit ec94989

Please sign in to comment.