update: document and functional code (#23)

* update: add documentation to newly added contents Signed-off-by: Gaukas Wang <[email protected]> * update: minor improvements Signed-off-by: Gaukas Wang <[email protected]> * deps: bump up caddy version Signed-off-by: Gaukas Wang <[email protected]> * fix: always set header to disable QUIC for TLS Also updated some wording to make error messages sound consistent. Signed-off-by: Gaukas Wang <[email protected]> * docs: update comments and README [ci skip] Signed-off-by: Gaukas Wang <[email protected]> * fix: save QUIC visitor when H3 is enabled only otherwise the TLS-fallback will incorrectly overwrite the cached QUIC fingerprint's sender. Signed-off-by: Gaukas Wang <[email protected]> * logging: fix typo and add debugging printouts Signed-off-by: Gaukas Wang <[email protected]> * update: new Caddyfile Signed-off-by: Gaukas Wang <[email protected]> --------- Signed-off-by: Gaukas Wang <[email protected]>
gaukas · Jun 6, 2024 · 560e27c · 560e27c
1 parent 3f06ffa
commit 560e27c
Show file tree

Hide file tree

Showing 26 changed files with 680 additions and 801 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -14,12 +14,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
-        go-version: "1.20"
+        go-version: "1.22.x"
 
     - name: Build
       run: go build -v ./...

diff --git a/README.md b/README.md
@@ -1,24 +1,32 @@
-# clienthellod
+# `clienthellod`: TLS ClientHello/QUIC Initial Packet reflection service
 ![Go Build Status](https://github.com/gaukas/clienthellod/actions/workflows/go.yml/badge.svg)
 [![Go Report Card](https://goreportcard.com/badge/github.com/gaukas/clienthellod)](https://goreportcard.com/report/github.com/gaukas/clienthellod)
-[![DeepSource](https://app.deepsource.com/gh/gaukas/clienthellod.svg/?label=active+issues&show_trend=true&token=GugDSBnYAxAF25QNpfyAO5d2)](https://app.deepsource.com/gh/gaukas/clienthellod/)
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fgaukas%2Fclienthellod.svg?type=shield&issueType=license)](https://app.fossa.com/projects/git%2Bgithub.com%2Fgaukas%2Fclienthellod?ref=badge_shield&issueType=license)
+[![Go Doc](https://pkg.go.dev/badge/github.com/refraction-networking/water.svg)](https://pkg.go.dev/github.com/refraction-networking/water)
 
-ClientHello Parser/Resolver as a Service from [tlsfingerprint.io](https://tlsfingerprint.io).
+`clienthellod`, read as "client-hello-D", is a TLS ClientHello/QUIC Initial Packet reflection service. It can be used to parses TLS ClientHello messages and QUIC Initial Packets into human-readable and highly programmable formats such as JSON. 
 
-## What does it do
-
-`clienthellod`, read as "client hello DEE", is a service that parses and resolves the ClientHello message sent by the client to the server. It is a part of the TLS fingerprintability research project which spans [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io). It parses the ClientHello messages sent by TLS clients and QUIC Client Initial Packets sent by QUIC clients and display the parsed information in a human-readable format with high programmability. 
+Is is a part of the TLS fingerprintability research project which spans [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io). It parses the ClientHello messages sent by TLS clients and QUIC Client Initial Packets sent by QUIC clients and display the parsed information in a human-readable format with high programmability. 
 
 See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](https://quic.tlsfingerprint.io) for more details about the project.
 
-## How to use
+## Quick Start
+
+`clienthellod` comes as a Go library, which can be used to parse both TLS and QUIC protocols. 
+
+### TLS/QUIC Fingerprinter
+
+```go
+    tlsFingerprinter := clienthellod.NewTLSFingerprinter()
+```
 
-`clienthellod` is provided as a Go library in the root directory of this repository. 
+```go
+    quicFingerprinter := clienthellod.NewQUICFingerprinter()
+```
 
-### Quick Start
+### TLS ClientHello
 
-#### TLS ClientHello
+#### From a `net.Conn`
 
 ```go
     tcpLis, err := net.Listen("tcp", ":443")
@@ -30,7 +38,7 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
 	}
     defer conn.Close()
 
-	ch, err := clienthellod.ReadClientHello(conn) // saves ClientHello
+	ch, err := clienthellod.ReadClientHello(conn) // reads ClientHello from the connection
     if err != nil {
         panic(err)
     }
@@ -46,11 +54,24 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
     }
 
     fmt.Println(string(jsonB))
-    fmt.Println("ClientHello ID: " + ch.FingerprintID(false)) // prints ClientHello's original fingerprint ID, as TLS extension IDs in their provided order
-    fmt.Println("ClientHello NormID: " + ch.FingerprintID(true)) // prints ClientHello's normalized fingerprint ID, as TLS extension IDs in a sorted order
+    fmt.Println("ClientHello ID: " + ch.HexID) // prints ClientHello's original fingerprint ID calculated using observed TLS extension order
+    fmt.Println("ClientHello NormID: " + ch.NormHexID) // prints ClientHello's normalized fingerprint ID calculated using sorted TLS extension list
 ```
 
-#### QUIC Client Initial Packet
+#### From raw `[]byte`
+
+```go
+    ch, err := clienthellod.UnmarshalClientHello(raw)
+    if err != nil {
+        panic(err)
+    }
+
+    // err := ch.ParseClientHello() // no need to call again, UnmarshalClientHello automatically calls ParseClientHello
+```
+
+### QUIC Initial Packets (Client-sourced)
+
+#### Single packet
 
 ```go
     udpConn, err := net.ListenUDP("udp", ":443")
@@ -62,7 +83,7 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
         panic(err)
     }
 
-    cip, err := clienthellod.ParseQUICCIP(buf[:n]) // reads in and parses QUIC Client Initial Packet
+    ci, err := clienthellod.UnmarshalQUICClientInitialPacket(buf[:n]) // decodes QUIC Client Initial Packet
     if err != nil {
         panic(err)    
     }
@@ -75,79 +96,42 @@ See [tlsfingerprint.io](https://tlsfingerprint.io) and [quic.tlsfingerprint.io](
     fmt.Println(string(jsonB)) // including fingerprint IDs of: ClientInitialPacket, QUIC Header, QUIC ClientHello, QUIC Transport Parameters' combination
 ```
 
-#### Use with Caddy
-
-`clienthellod` is also provided as a Caddy plugin, `modcaddy`, which can be used to capture ClientHello messages and QUIC Client Initial Packets. See Section [modcaddy](#modcaddy) for more details.
-
-## modcaddy
-
-`modcaddy` is a Caddy plugin that provides:
-- An caddy `app` that can be used to temporarily store captured ClientHello messages and QUIC Client Initial Packets. 
-- A caddy `handler` that can be used to serve the ClientHello messages and QUIC Client Initial Packets to the client sending the request. 
-- A caddy `listener` that can be used to capture ClientHello messages and QUIC Client Initial Packets.
+#### Multiple packets
 
-You will need to use [xcaddy](https://github.com/caddyserver/xcaddy) to rebuild Caddy with `modcaddy` included.
+Implementations including Chrome/Chromium sends oversized Client Hello which does not fit into one single QUIC packet, in which case multiple QUIC Initial Packets are sent.
 
-It is worth noting that some web browsers may not choose to switch to QUIC protocol in localhost environment, which may result in the QUIC Client Initial Packet not being sent and therefore not being captured/analyzed.
-
-### Build 
-
-```bash
-xcaddy build --with github.com/gaukas/clienthellod/modcaddy
-```
-
-#### When build locally with changes 
+```go
+    gci := GatherClientInitials() // Each GatherClientInitials reassembles one QUIC Client Initial Packets stream. Use a QUIC Fingerprinter for multiple potential senders, which automatically demultiplexes the packets based on the source address.
+
+    udpConn, err := net.ListenUDP("udp", ":443")
+    defer udpConn.Close()
 
-```bash
-xcaddy build --with github.com/gaukas/clienthellod/modcaddy --with github.com/gaukas/clienthellod/=./
-```
+    for {
+        buf := make([]byte, 65535)
+        n, addr, err := udpConn.ReadFromUDP(buf)
+        if err != nil {
+            panic(err)
+        }
 
-### Caddyfile
+        if addr != knownSenderAddr {
+            continue
+        }
 
-A sample Caddyfile is provided below.
+        ci, err := clienthellod.UnmarshalQUICClientInitialPacket(buf[:n]) // decodes QUIC Client Initial Packet
+        if err != nil {
+            panic(err)    
+        }
 
-```Caddyfile
-{      
-    # debug # for debugging purpose
-    # https_port   443 # currently, QUIC listener works only on port 443, otherwise you need to make changes to the code
-    order clienthellod before file_server # make sure it hits handler before file_server
-    clienthellod { # app (reservoir)
-        validfor 120s 30s # params: validFor [cleanEvery] # increased for QUIC
-    }
-    servers {
-        listener_wrappers {
-            clienthellod { # listener
-                tcp # listens for TCP and saves TLS ClientHello 
-                udp # listens for UDP and saves QUIC Client Initial Packet
-            }
-            tls
+        err = gci.AddPacket(ci)
+        if err != nil {
+            panic(err)
         }
-        # protocols h3
     }
-}
+```
 
-1.mydomain.com {
-    # tls internal
-    clienthellod { # handler
-        # quic # mutually exclusive with tls
-        tls # listener_wrappers.clienthellod.tcp must be set
-    }
-    file_server {
-        root /var/www/html
-    }
-}
+### Use with Caddy
 
-2.mydomain.com {
-    # tls internal
-    clienthellod { # handler
-        quic # listener_wrappers.clienthellod.udp must be set
-        # tls # mutually exclusive with quic
-    }
-    file_server {
-        root /var/www/html
-    }
-}
-```
+We also provide clienthellod as a Caddy Module in `modcaddy`, which you can use with Caddy to capture ClientHello messages and QUIC Client Initial Packets. See [modcaddy](https://github.com/gaukas/clienthellod/tree/master/modcaddy) for more details.
 
 ## License
 

diff --git a/clienthello.go b/clienthello.go
@@ -15,6 +15,7 @@ import (
 	"golang.org/x/crypto/cryptobyte"
 )
 
+// ClientHello represents a captured ClientHello message with all fingerprintable fields.
 type ClientHello struct {
 	raw []byte
 
@@ -53,7 +54,7 @@ type ClientHello struct {
 	lengthPrefixedCertCompressAlgos []uint8
 	keyshareGroupsWithLengths       []uint16
 
-	// QUIC-only
+	// QUIC-only, nil if not QUIC
 	qtp *QUICTransportParameters
 }
 

diff --git a/fingerprint_hash.go b/fingerprint_hash.go
@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"hash"
-	"sort"
 
 	"github.com/gaukas/clienthellod/internal/utils"
 )
@@ -23,16 +22,17 @@ func updateU64(h hash.Hash, i uint64) {
 	binary.Write(h, binary.BigEndian, i)
 }
 
+// FingerprintID is the type of fingerprint ID.
 type FingerprintID int64
 
+// AsHex returns the hex representation of this fingerprint ID.
 func (id FingerprintID) AsHex() string {
 	hid := make([]byte, 8)
 	binary.BigEndian.PutUint64(hid, uint64(id))
 	return hex.EncodeToString(hid)
 }
 
-// FingerprintNID calculates fingerprint Numeric ID of ClientHello.
-// Fingerprint is defined by
+// calcNumericID returns the numeric ID of this client hello.
 func (ch *ClientHello) calcNumericID() (orig, norm int64) {
 	for _, normalized := range []bool{false, true} {
 		h := sha1.New() // skipcq: GO-S1025, GSC-G401,
@@ -66,6 +66,7 @@ func (ch *ClientHello) calcNumericID() (orig, norm int64) {
 	return
 }
 
+// calcNumericID returns the numeric ID of this gathered client initial.
 func (gci *GatheredClientInitials) calcNumericID() uint64 {
 	h := sha1.New() // skipcq: GO-S1025, GSC-G401
 	updateArr(h, gci.Packets[0].Header.Version)
@@ -79,9 +80,6 @@ func (gci *GatheredClientInitials) calcNumericID() uint64 {
 		allFrameIDs = append(allFrameIDs, p.frames.FrameTypesUint8()...)
 	}
 	dedupAllFrameIDs := utils.DedupIntArr(allFrameIDs)
-	sort.Slice(dedupAllFrameIDs, func(i, j int) bool {
-		return dedupAllFrameIDs[i] < dedupAllFrameIDs[j]
-	})
 	updateArr(h, dedupAllFrameIDs)
 
 	if gci.Packets[0].Header.HasToken {
@@ -93,7 +91,7 @@ func (gci *GatheredClientInitials) calcNumericID() uint64 {
 	return binary.BigEndian.Uint64(h.Sum(nil)[0:8])
 }
 
-// NID returns the numeric ID of this transport parameters combination.
+// calcNumericID returns the numeric ID of this transport parameters combination.
 func (qtp *QUICTransportParameters) calcNumericID() uint64 {
 	h := sha1.New() // skipcq: GO-S1025, GSC-G401
 	updateArr(h, qtp.MaxIdleTimeout)