Skip to content

Commit

Permalink
完成明细记录写入
Browse files Browse the repository at this point in the history
  • Loading branch information
@steden
steden committed Dec 25, 2023
1 parent f102fa3 commit 45cce16
Show file tree
Hide file tree
Showing 8 changed files with 181 additions and 90 deletions.
16 changes: 8 additions & 8 deletions driver/clickhouse/chContext.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ var CHContext *chContext

// EsContext 链路追踪上下文
type chContext struct {
TraceContext data.TableSet[TraceContextPO] `data:"name=link_trace;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id) PARTITION BY toYYYYMM(app_name)"`
TraceDetailDatabase data.TableSet[TraceDetailDatabasePO] `data:"name=trace_detail_database;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,db_name,table_name,connection_string) PARTITION BY toYYYYMM(app_name)"`
TraceDetailEs data.TableSet[TraceDetailEsPO] `data:"name=trace_detail_es;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,index_name,aliases_name) PARTITION BY toYYYYMM(app_name)"`
TraceDetailEtcd data.TableSet[TraceDetailEtcdPO] `data:"name=trace_detail_etcd;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,key) PARTITION BY toYYYYMM(app_name)"`
TraceDetailHand data.TableSet[TraceDetailHandPO] `data:"name=trace_detail_hand;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,name) PARTITION BY toYYYYMM(app_name)"`
TraceDetailHttp data.TableSet[TraceDetailHttpPO] `data:"name=trace_detail_http;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,method,url) PARTITION BY toYYYYMM(app_name)"`
TraceDetailMq data.TableSet[TraceDetailMqPO] `data:"name=trace_detail_mq;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,server,exchange,routing_key) PARTITION BY toYYYYMM(app_name)"`
TraceDetailRedis data.TableSet[TraceDetailRedisPO] `data:"name=trace_detail_redis;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,key,field) PARTITION BY toYYYYMM(app_name)"`
TraceContext data.TableSet[TraceContextPO] `data:"name=link_trace;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id) PARTITION BY app_name"`
TraceDetailDatabase data.TableSet[TraceDetailDatabasePO] `data:"name=trace_detail_database;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,db_name,table_name,connection_string) PARTITION BY app_name"`
TraceDetailEs data.TableSet[TraceDetailEsPO] `data:"name=trace_detail_es;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,index_name,aliases_name) PARTITION BY app_name"`
TraceDetailEtcd data.TableSet[TraceDetailEtcdPO] `data:"name=trace_detail_etcd;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,key) PARTITION BY app_name"`
TraceDetailHand data.TableSet[TraceDetailHandPO] `data:"name=trace_detail_hand;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,name) PARTITION BY app_name"`
TraceDetailHttp data.TableSet[TraceDetailHttpPO] `data:"name=trace_detail_http;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,method,url) PARTITION BY app_name"`
TraceDetailMq data.TableSet[TraceDetailMqPO] `data:"name=trace_detail_mq;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,server,exchange,routing_key) PARTITION BY app_name"`
TraceDetailRedis data.TableSet[TraceDetailRedisPO] `data:"name=trace_detail_redis;migrate=ReplacingMergeTree() ORDER BY (app_name,parent_app_name,app_ip,app_id,trace_id,key,field) PARTITION BY app_name"`
}

// initCHContext 初始化上下文
Expand Down
59 changes: 52 additions & 7 deletions driver/clickhouse/saveTraceContextConsumer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,25 +36,70 @@ func saveTraceContextConsumer(subscribeName string, lstMessage collections.ListA
if !traceContext.WatchKey.IsNil() {
po.WatchKey = &traceContext.WatchKey
}
if !traceContext.Exception.IsNil() {
po.Exception = &traceContext.Exception
}
lstTraceContext.Add(po)

// 明细
for _, detail := range traceContext.List {
switch detailType := detail.(type) {
case *linkTrace.TraceDetailDatabase:
lstTraceDetailDatabase.Add(mapper.Single[TraceDetailDatabasePO](*detailType))
databasePO := mapper.Single[TraceDetailDatabasePO](*detailType)
databasePO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &databasePO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
databasePO.Exception = &detailType.Exception
}
lstTraceDetailDatabase.Add(databasePO)
case *linkTrace.TraceDetailEs:
lstTraceDetailEs.Add(mapper.Single[TraceDetailEsPO](*detailType))
esPO := mapper.Single[TraceDetailEsPO](*detailType)
esPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &esPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
esPO.Exception = &detailType.Exception
}
lstTraceDetailEs.Add(esPO)
case *linkTrace.TraceDetailEtcd:
lstTraceDetailEtcd.Add(mapper.Single[TraceDetailEtcdPO](*detailType))
etcdPO := mapper.Single[TraceDetailEtcdPO](*detailType)
etcdPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &etcdPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
etcdPO.Exception = &detailType.Exception
}
lstTraceDetailEtcd.Add(etcdPO)
case *linkTrace.TraceDetailHand:
lstTraceDetailHand.Add(mapper.Single[TraceDetailHandPO](*detailType))
handPO := mapper.Single[TraceDetailHandPO](*detailType)
handPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &handPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
handPO.Exception = &detailType.Exception
}
lstTraceDetailHand.Add(handPO)
case *linkTrace.TraceDetailHttp:
lstTraceDetailHttp.Add(mapper.Single[TraceDetailHttpPO](*detailType))
httpPO := mapper.Single[TraceDetailHttpPO](*detailType)
httpPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &httpPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
httpPO.Exception = &detailType.Exception
}
lstTraceDetailHttp.Add(httpPO)
case *linkTrace.TraceDetailMq:
lstTraceDetailMq.Add(mapper.Single[TraceDetailMqPO](*detailType))
mqPO := mapper.Single[TraceDetailMqPO](*detailType)
mqPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &mqPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
mqPO.Exception = &detailType.Exception
}
lstTraceDetailMq.Add(mqPO)
case *linkTrace.TraceDetailRedis:
lstTraceDetailRedis.Add(mapper.Single[TraceDetailRedisPO](*detailType))
redisPO := mapper.Single[TraceDetailRedisPO](*detailType)
redisPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &redisPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
redisPO.Exception = &detailType.Exception
}
lstTraceDetailRedis.Add(redisPO)
}
}
})
Expand Down
4 changes: 2 additions & 2 deletions driver/clickhouse/traceContextPO.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ type TraceContextPO struct {
AppName string `gorm:"not null;default:'';comment:应用名称"`
AppIp string `gorm:"not null;default:'';comment:应用IP"`
ParentAppName string `gorm:"not null;default:'';comment:上游应用"`
StartTs int64 `gorm:"not null;default:0;comment:调用开始时间戳"`
StartTs int64 `gorm:"not null;default:0;comment:调用开始时间戳（微秒）"`
EndTs int64 `gorm:"not null;default:0;comment:调用结束时间戳"`
UseTs time.Duration `gorm:"not null;default:0;comment:总共使用时间毫秒"`
UseTs time.Duration `gorm:"not null;default:0;comment:总共使用时间微秒"`
TraceType eumTraceType.Enum `gorm:"not null;comment:状态码"`
Exception *trace.ExceptionStack `gorm:"json;not null;comment:异常信息"`
List []trace.ITraceDetail `gorm:"json;not null;comment:调用的上下文"`
Expand Down
80 changes: 40 additions & 40 deletions driver/clickhouse/traceDetailPO.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,62 +7,62 @@ import (
)

type BaseTraceDetailPO struct {
TraceId int64 `gorm:"not null;default:0;comment:上下文ID"`
AppId int64 `gorm:"not null;default:0;comment:应用ID"`
AppName string `gorm:"not null;default:'';comment:应用名称"`
AppIp string `gorm:"not null;default:'';comment:应用IP"`
ParentAppName string `gorm:"not null;default:'';comment:上游应用"`
DetailId int64 `gorm:"not null;default:0;comment:明细ID"`
ParentDetailId int64 `gorm:"not null;default:0;comment:父级明细ID"`
Level int `gorm:"not null;comment:当前层级（入口为0层）"`
MethodName string `gorm:"not null;default:'';comment:调用方法"`
CallType eumCallType.Enum `gorm:"not null;comment:调用类型"`
Timeline time.Duration `gorm:"not null;default:0;comment:从入口开始统计"`
UnTraceTs time.Duration `gorm:"not null;default:0;comment:上一次结束到现在开始之间未Trace的时间"`
StartTs int64 `gorm:"not null;default:0;comment:调用开始时间戳"`
EndTs int64 `gorm:"not null;default:0;comment:调用停止时间戳"`
UseTs time.Duration `gorm:"not null;default:0;comment:总共使用时间毫秒"`
Exception trace.ExceptionStack `gorm:"json;not null;comment:异常信息"`
TraceId int64 `gorm:"not null;default:0;comment:上下文ID"`
AppId int64 `gorm:"not null;default:0;comment:应用ID"`
AppName string `gorm:"not null;default:'';comment:应用名称"`
AppIp string `gorm:"not null;default:'';comment:应用IP"`
ParentAppName string `gorm:"not null;default:'';comment:上游应用"`
DetailId int64 `gorm:"not null;default:0;comment:明细ID"`
ParentDetailId int64 `gorm:"not null;default:0;comment:父级明细ID"`
Level int `gorm:"not null;comment:当前层级（入口为0层）"`
MethodName string `gorm:"not null;default:'';comment:调用方法"`
CallType eumCallType.Enum `gorm:"not null;comment:调用类型"`
Timeline time.Duration `gorm:"not null;default:0;comment:从入口开始统计（微秒）"`
UnTraceTs time.Duration `gorm:"not null;default:0;comment:上一次结束到现在开始之间未Trace的时间（微秒）"`
StartTs int64 `gorm:"not null;default:0;comment:调用开始时间戳（微秒）"`
EndTs int64 `gorm:"not null;default:0;comment:调用停止时间戳（微秒）"`
UseTs time.Duration `gorm:"not null;default:0;comment:总共使用时间微秒"`
Exception *trace.ExceptionStack `gorm:"json;not null;comment:异常信息"`
}

type TraceDetailDatabasePO struct {
BaseTraceDetailPO
DbName string `gorm:"not null;default:'';comment:数据库名"`
TableName string `gorm:"not null;default:'';comment:表名"`
Sql string `gorm:"not null;default:'';comment:SQL"`
ConnectionString string `gorm:"not null;default:'';comment:连接字符串"`
RowsAffected int64 `gorm:"not null;default:0;comment:影响行数"`
BaseTraceDetailPO `gorm:"embedded"`
DbName string `gorm:"not null;default:'';comment:数据库名"`
TableName string `gorm:"not null;default:'';comment:表名"`
Sql string `gorm:"not null;default:'';comment:SQL"`
ConnectionString string `gorm:"not null;default:'';comment:连接字符串"`
RowsAffected int64 `gorm:"not null;default:0;comment:影响行数"`
}

type TraceDetailEsPO struct {
trace.BaseTraceDetail
IndexName string `gorm:"not null;default:'';comment:索引名称"`
AliasesName string `gorm:"not null;default:'';comment:别名"`
BaseTraceDetailPO `gorm:"embedded"`
IndexName string `gorm:"not null;default:'';comment:索引名称"`
AliasesName string `gorm:"not null;default:'';comment:别名"`
}
type TraceDetailEtcdPO struct {
trace.BaseTraceDetail
Key string `gorm:"not null;default:'';comment:etcd key"`
LeaseID int64 `gorm:"not null;default:0;comment:LeaseID"`
BaseTraceDetailPO `gorm:"embedded"`
Key string `gorm:"not null;default:'';comment:etcd key"`
LeaseID int64 `gorm:"not null;default:0;comment:LeaseID"`
}

// TraceDetailHandPO 手动埋点
type TraceDetailHandPO struct {
trace.BaseTraceDetail
Name string `gorm:"not null;default:'';comment:名称"`
BaseTraceDetailPO `gorm:"embedded"`
Name string `gorm:"not null;default:'';comment:名称"`
}
type TraceDetailHttpPO struct {
trace.BaseTraceDetail
Method string `gorm:"not null;default:'';comment:post/get/put/delete"`
Url string `gorm:"not null;default:'';comment:请求url"`
BaseTraceDetailPO `gorm:"embedded"`
Method string `gorm:"not null;default:'';comment:post/get/put/delete"`
Url string `gorm:"not null;default:'';comment:请求url"`
}
type TraceDetailMqPO struct {
trace.BaseTraceDetail
Server string `gorm:"not null;default:'';comment:MQ服务器地址"`
Exchange string `gorm:"not null;default:'';comment:交换器名称"`
RoutingKey string `gorm:"not null;default:'';comment:路由key"`
BaseTraceDetailPO `gorm:"embedded"`
Server string `gorm:"not null;default:'';comment:MQ服务器地址"`
Exchange string `gorm:"not null;default:'';comment:交换器名称"`
RoutingKey string `gorm:"not null;default:'';comment:路由key"`
}
type TraceDetailRedisPO struct {
trace.BaseTraceDetail
Key string `gorm:"not null;default:'';comment:redis key"`
Field string `gorm:"not null;default:'';comment:hash field"`
BaseTraceDetailPO `gorm:"embedded"`
Key string `gorm:"not null;default:'';comment:redis key"`
Field string `gorm:"not null;default:'';comment:hash field"`
}
60 changes: 53 additions & 7 deletions driver/elasticSearch/saveTraceContextConsumer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,74 @@ func saveTraceContextConsumer(subscribeName string, lstMessage collections.ListA
if !traceContext.WatchKey.IsNil() {
po.WatchKey = &traceContext.WatchKey
}
if !traceContext.Exception.IsNil() {
po.Exception = &traceContext.Exception
}
lstTraceContext.Add(po)

// 明细
for _, detail := range traceContext.List {
switch detailType := detail.(type) {
case *linkTrace.TraceDetailDatabase:
lstTraceDetailDatabase.Add(mapper.Single[TraceDetailDatabasePO](*detailType))
databasePO := mapper.Single[TraceDetailDatabasePO](*detailType)
databasePO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &databasePO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
databasePO.Exception = &detailType.Exception
}
lstTraceDetailDatabase.Add(databasePO)
case *linkTrace.TraceDetailEs:
lstTraceDetailEs.Add(mapper.Single[TraceDetailEsPO](*detailType))
esPO := mapper.Single[TraceDetailEsPO](*detailType)
esPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &esPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
esPO.Exception = &detailType.Exception
}
lstTraceDetailEs.Add(esPO)
case *linkTrace.TraceDetailEtcd:
lstTraceDetailEtcd.Add(mapper.Single[TraceDetailEtcdPO](*detailType))
etcdPO := mapper.Single[TraceDetailEtcdPO](*detailType)
etcdPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &etcdPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
etcdPO.Exception = &detailType.Exception
}
lstTraceDetailEtcd.Add(etcdPO)
case *linkTrace.TraceDetailHand:
lstTraceDetailHand.Add(mapper.Single[TraceDetailHandPO](*detailType))
handPO := mapper.Single[TraceDetailHandPO](*detailType)
handPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &handPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
handPO.Exception = &detailType.Exception
}
lstTraceDetailHand.Add(handPO)
case *linkTrace.TraceDetailHttp:
lstTraceDetailHttp.Add(mapper.Single[TraceDetailHttpPO](*detailType))
httpPO := mapper.Single[TraceDetailHttpPO](*detailType)
httpPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &httpPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
httpPO.Exception = &detailType.Exception
}
lstTraceDetailHttp.Add(httpPO)
case *linkTrace.TraceDetailMq:
lstTraceDetailMq.Add(mapper.Single[TraceDetailMqPO](*detailType))
mqPO := mapper.Single[TraceDetailMqPO](*detailType)
mqPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &mqPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
mqPO.Exception = &detailType.Exception
}
lstTraceDetailMq.Add(mqPO)
case *linkTrace.TraceDetailRedis:
lstTraceDetailRedis.Add(mapper.Single[TraceDetailRedisPO](*detailType))
redisPO := mapper.Single[TraceDetailRedisPO](*detailType)
redisPO.BaseTraceDetailPO = mapper.Single[BaseTraceDetailPO](detailType.BaseTraceDetail)
_ = mapper.Auto(traceContext, &redisPO.BaseTraceDetailPO)
if !detailType.Exception.IsNil() {
redisPO.Exception = &detailType.Exception
}
lstTraceDetailRedis.Add(redisPO)
}
}
})
// 写入上下文
err := ESContext.TraceContext.InsertList(lstTraceContext)
flog.ErrorIfExists(err)

Expand Down
4 changes: 2 additions & 2 deletions driver/elasticSearch/traceContextPO.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ type TraceContextPO struct {
AppName string // 应用名称
AppIp string // 应用IP
ParentAppName string // 上游应用
StartTs int64 // 调用开始时间戳
StartTs int64 // 调用开始时间戳（微秒）
EndTs int64 // 调用结束时间戳
UseTs time.Duration // 总共使用时间毫秒
UseTs time.Duration // 总共使用时间微秒
TraceType eumTraceType.Enum // 状态码
List []trace.ITraceDetail `es_type:"object"` // 调用的上下文
Exception *trace.ExceptionStack `es_type:"object"` // 异常信息
Expand Down
Loading

0 comments on commit 45cce16

Please sign in to comment.