Skip to content

Commit

Permalink
Handle all the datatypes in the EMA sampler (#186)
Browse files Browse the repository at this point in the history
Now that we support msgpack the types that might be in the data hash are more varied than they used to be. 

Adds the ability to format any type as a string in order to handle this
  • Loading branch information
martin308 authored Oct 22, 2020
1 parent a70ff25 commit 79a353b
Show file tree
Hide file tree
Showing 5 changed files with 272 additions and 148 deletions.
76 changes: 4 additions & 72 deletions sample/dynamic.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package sample

import (
"math/rand"
"sort"
"strconv"

dynsampler "github.com/honeycombio/dynsampler-go"

Expand All @@ -20,12 +18,10 @@ type DynamicSampler struct {

sampleRate int64
clearFrequencySec int64
fieldList []string
useTraceLength bool
addDynsampleKey bool
addDynsampleField string
configName string

key *traceKey

dynsampler dynsampler.Sampler
}

Expand All @@ -37,18 +33,7 @@ func (d *DynamicSampler) Start() error {
d.Config.ClearFrequencySec = 30
}
d.clearFrequencySec = d.Config.ClearFrequencySec

// get list of fields to use when constructing the dynsampler key
fieldList := d.Config.FieldList

// always put the field list in sorted order for easier comparison
sort.Strings(fieldList)
d.fieldList = fieldList

d.useTraceLength = d.Config.UseTraceLength

d.addDynsampleKey = d.Config.AddSampleRateKeyToTrace
d.addDynsampleField = d.Config.AddSampleRateKeyToTraceField
d.key = newTraceKey(d.Config.FieldList, d.Config.UseTraceLength, d.Config.AddSampleRateKeyToTrace, d.Config.AddSampleRateKeyToTraceField)

// spin up the actual dynamic sampler
d.dynsampler = &dynsampler.AvgSampleRate{
Expand All @@ -66,7 +51,7 @@ func (d *DynamicSampler) Start() error {
}

func (d *DynamicSampler) GetSampleRate(trace *types.Trace) (uint, bool) {
key := d.buildKey(trace)
key := d.key.buildAndAdd(trace)
rate := d.dynsampler.GetSampleRate(key)
if rate < 1 { // protect against dynsampler being broken even though it shouldn't be
rate = 1
Expand All @@ -86,56 +71,3 @@ func (d *DynamicSampler) GetSampleRate(trace *types.Trace) (uint, bool) {
d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
return uint(rate), shouldKeep
}

// buildKey takes a trace and returns the key to use for the dynsampler.
func (d *DynamicSampler) buildKey(trace *types.Trace) string {
// fieldCollector gets all values from the fields listed in the config, even
// if they happen multiple times.
fieldCollector := map[string][]string{}

// for each field, for each span, get the value of that field
spans := trace.GetSpans()
for _, field := range d.fieldList {
for _, span := range spans {
if val, ok := span.Data[field]; ok {
switch val := val.(type) {
case string:
fieldCollector[field] = append(fieldCollector[field], val)
case float64:
valStr := strconv.FormatFloat(val, 'f', -1, 64)
fieldCollector[field] = append(fieldCollector[field], valStr)
case bool:
valStr := strconv.FormatBool(val)
fieldCollector[field] = append(fieldCollector[field], valStr)
}
}
}
}
// ok, now we have a map of fields to a list of all values for that field.

var key string
for _, field := range d.fieldList {
// sort and collapse list
sort.Strings(fieldCollector[field])
var prevStr string
for _, str := range fieldCollector[field] {
if str != prevStr {
key += str + "•"
}
prevStr = str
}
// get ready for the next element
key += ","
}
if d.useTraceLength {
key += strconv.FormatInt(int64(len(spans)), 10)
}

if d.addDynsampleKey {
for _, span := range trace.GetSpans() {
span.Data[d.addDynsampleField] = key
}
}

return key
}
76 changes: 4 additions & 72 deletions sample/dynamic_ema.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package sample

import (
"math/rand"
"sort"
"strconv"

dynsampler "github.com/honeycombio/dynsampler-go"

Expand All @@ -25,12 +23,10 @@ type EMADynamicSampler struct {
burstMultiple float64
burstDetectionDelay uint
maxKeys int
fieldList []string
useTraceLength bool
addDynsampleKey bool
addDynsampleField string
configName string

key *traceKey

dynsampler dynsampler.Sampler
}

Expand All @@ -44,18 +40,7 @@ func (d *EMADynamicSampler) Start() error {
d.burstMultiple = d.Config.BurstMultiple
d.burstDetectionDelay = d.Config.BurstDetectionDelay
d.maxKeys = d.Config.MaxKeys

// get list of fields to use when constructing the dynsampler key
fieldList := d.Config.FieldList

// always put the field list in sorted order for easier comparison
sort.Strings(fieldList)
d.fieldList = fieldList

d.useTraceLength = d.Config.UseTraceLength

d.addDynsampleKey = d.Config.AddSampleRateKeyToTrace
d.addDynsampleField = d.Config.AddSampleRateKeyToTraceField
d.key = newTraceKey(d.Config.FieldList, d.Config.UseTraceLength, d.Config.AddSampleRateKeyToTrace, d.Config.AddSampleRateKeyToTraceField)

// spin up the actual dynamic sampler
d.dynsampler = &dynsampler.EMASampleRate{
Expand All @@ -78,7 +63,7 @@ func (d *EMADynamicSampler) Start() error {
}

func (d *EMADynamicSampler) GetSampleRate(trace *types.Trace) (uint, bool) {
key := d.buildKey(trace)
key := d.key.buildAndAdd(trace)
rate := d.dynsampler.GetSampleRate(key)
if rate < 1 { // protect against dynsampler being broken even though it shouldn't be
rate = 1
Expand All @@ -98,56 +83,3 @@ func (d *EMADynamicSampler) GetSampleRate(trace *types.Trace) (uint, bool) {
d.Metrics.Histogram("dynsampler_sample_rate", float64(rate))
return uint(rate), shouldKeep
}

// buildKey takes a trace and returns the key to use for the dynsampler.
func (d *EMADynamicSampler) buildKey(trace *types.Trace) string {
// fieldCollector gets all values from the fields listed in the config, even
// if they happen multiple times.
fieldCollector := map[string][]string{}

// for each field, for each span, get the value of that field
spans := trace.GetSpans()
for _, field := range d.fieldList {
for _, span := range spans {
if val, ok := span.Data[field]; ok {
switch val := val.(type) {
case string:
fieldCollector[field] = append(fieldCollector[field], val)
case float64:
valStr := strconv.FormatFloat(val, 'f', -1, 64)
fieldCollector[field] = append(fieldCollector[field], valStr)
case bool:
valStr := strconv.FormatBool(val)
fieldCollector[field] = append(fieldCollector[field], valStr)
}
}
}
}
// ok, now we have a map of fields to a list of all values for that field.

var key string
for _, field := range d.fieldList {
// sort and collapse list
sort.Strings(fieldCollector[field])
var prevStr string
for _, str := range fieldCollector[field] {
if str != prevStr {
key += str + "•"
}
prevStr = str
}
// get ready for the next element
key += ","
}
if d.useTraceLength {
key += strconv.FormatInt(int64(len(spans)), 10)
}

if d.addDynsampleKey {
for _, span := range trace.GetSpans() {
span.Data[d.addDynsampleField] = key
}
}

return key
}
15 changes: 11 additions & 4 deletions sample/dynamic_ema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func TestDynamicEMAAddSampleRateKeyToTrace(t *testing.T) {

sampler := &EMADynamicSampler{
Config: &config.EMADynamicSamplerConfig{
FieldList: []string{"http.status_code"},
FieldList: []string{"http.status_code", "request.path", "app.team.id", "important_field"},
AddSampleRateKeyToTrace: true,
AddSampleRateKeyToTraceField: "meta.key",
},
Expand All @@ -34,7 +34,10 @@ func TestDynamicEMAAddSampleRateKeyToTrace(t *testing.T) {
trace.AddSpan(&types.Span{
Event: types.Event{
Data: map[string]interface{}{
"http.status_code": "200",
"http.status_code": 200,
"app.team.id": float64(4),
"important_field": true,
"request.path": "/{slug}/fun",
},
},
})
Expand All @@ -43,11 +46,15 @@ func TestDynamicEMAAddSampleRateKeyToTrace(t *testing.T) {
sampler.GetSampleRate(trace)

spans := trace.GetSpans()

assert.Len(t, spans, spanCount, "should have the same number of spans as input")
for _, span := range spans {
assert.Equal(t, span.Event.Data, map[string]interface{}{
"http.status_code": "200",
"meta.key": "200•,",
"http.status_code": 200,
"app.team.id": float64(4),
"important_field": true,
"request.path": "/{slug}/fun",
"meta.key": "4•,200•,true•,/{slug}/fun•,",
}, "should add the sampling key to all spans in the trace")
}
}
80 changes: 80 additions & 0 deletions sample/trace_key.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package sample

import (
"fmt"
"sort"
"strconv"

"github.com/honeycombio/refinery/types"
)

type traceKey struct {
fields []string
useTraceLength bool
addDynsampleKey bool
addDynsampleField string
}

func newTraceKey(fields []string, useTraceLength, addDynsampleKey bool, addDynsampleField string) *traceKey {
// always put the field list in sorted order for easier comparison
sort.Strings(fields)
return &traceKey{
fields: fields,
useTraceLength: useTraceLength,
addDynsampleKey: addDynsampleKey,
addDynsampleField: addDynsampleField,
}
}

// buildAndAdd, builds the trace key and adds it to the trace if configured to
// do so
func (d *traceKey) buildAndAdd(trace *types.Trace) string {
key := d.build(trace)

if d.addDynsampleKey {
for _, span := range trace.GetSpans() {
span.Data[d.addDynsampleField] = key
}
}

return key
}

// build, builds the trace key based on the configuration of the traceKeyGenerator
func (d *traceKey) build(trace *types.Trace) string {
// fieldCollector gets all values from the fields listed in the config, even
// if they happen multiple times.
fieldCollector := map[string][]string{}

// for each field, for each span, get the value of that field
spans := trace.GetSpans()
for _, field := range d.fields {
for _, span := range spans {
if val, ok := span.Data[field]; ok {
fieldCollector[field] = append(fieldCollector[field], fmt.Sprintf("%v", val))
}
}
}
// ok, now we have a map of fields to a list of all values for that field.

var key string
for _, field := range d.fields {
// sort and collapse list
sort.Strings(fieldCollector[field])
var prevStr string
for _, str := range fieldCollector[field] {
if str != prevStr {
key += str + "•"
}
prevStr = str
}
// get ready for the next element
key += ","
}

if d.useTraceLength {
key += strconv.FormatInt(int64(len(spans)), 10)
}

return key
}
Loading

0 comments on commit 79a353b

Please sign in to comment.