From c83a5fa662a5c989cd72b113788fe1eabc7d2c3e Mon Sep 17 00:00:00 2001 From: silenceper Date: Thu, 29 Feb 2024 16:49:40 +0800 Subject: [PATCH] =?UTF-8?q?--task=3D74868230=20=E8=8A=82=E7=82=B9NotReady?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8Ccni=20ipam=20=E5=8D=A1=E4=B8=BB?= =?UTF-8?q?=EF=BC=8C=E5=AF=BC=E8=87=B4=E8=8A=82=E7=82=B9NotReady=E9=97=AE?= =?UTF-8?q?=E9=A2=98[glog=E9=97=AE=E9=A2=98]=20(merge=20request=20!1636)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squash merge branch 'f-bcs-network-services' into 'master' **1、为什么会NotReady** PLEG 中同步pod状态 执行GetPodNetworkStatus 中lock和SetUpPod有锁，而ipam无返回导致lock一直被持有而无法释放，导致PLEG超时 **2、ipam为什么会卡主** 并发测试，当触发日志轮转，需要清理时，文件不存在；在glog中执行log.Printf 和flush日志 相互等待lock ipam测试命令 ``` echo '{ "cniVersion": "0.3.0", "name": "bcs-test-ip", "ipam": {"type": "bcs-netservice-ipam","host": "xxxx","netserviceEndpoint": "http://xxxx:8088","logDir": "/var/log/bcs-underlay-cni","routes": [{"dst":"0.0.0.0/0"}]} }' | CNI_COMMAND=ADD CNI_CONTAINERID=example CNI_NETNS=/dev/null CNI_IFNAME=dummy0 CNI_PATH=. CNI_ARGS=K8S_POD_NAMESPACE=test-nas\;K8S_POD_NAME=test-pod ./bcs-netservice-ipam ``` **ipam goroutine 信息如下： ** ``` goroutine profile: total 61 @ 0x4300d6 0x466f45 0x6d3bd5 0x6d39ed 0x6d078b 0x6dfa25 0x6e0573 0x6784cf 0x679e69 0x67ae76 0x677272 0x46d021# 0x466f44 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 /root/sdk/go1.20.2/src/runtime/mprof.go:844 # 0x6d3bd4 runtime/pprof.writeRuntimeProfile+0xb4 /root/sdk/go1.20.2/src/runtime/pprof/pprof.go:734 # 0x6d39ec runtime/pprof.writeGoroutine+0x4c /root/sdk/go1.20.2/src/runtime/pprof/pprof.go:694 # 0x6d078a runtime/pprof.(*Profile).WriteTo+0x14a /root/sdk/go1.20.2/src/runtime/pprof/pprof.go:329 # 0x6dfa24 net/http/pprof.handler.ServeHTTP+0x4a4 /root/sdk/go1.20.2/src/net/http/pprof/pprof.go:259 # 0x6e0572 net/http/pprof.Index+0xf2 /root/sdk/go1.20.2/src/net/http/pprof/pprof.go:376 # 0x6784ce net/http.HandlerFunc.ServeHTTP+0x2e /root/sdk/go1.20.2/src/net/http/server.go:2122 # 0x679e68 net/http.(*ServeMux).ServeHTTP+0x148 /root/sdk/go1.20.2/src/net/http/server.go:2500 # 0x67ae75 net/http.serverHandler.ServeHTTP+0x315 /root/sdk/go1.20.2/src/net/http/server.go:2936 # 0x677271 net/http.(*conn).serve+0x611 /root/sdk/go1.20.2/src/net/http/server.go:1995 1 @ 0x43ac56 0x4335b7 0x467369 0x4c3312 0x4c3ef9 0x4c3ee7 0x519009 0x524f65 0x67109f 0x46d021 # 0x467368 internal/poll.runtime_pollWait+0x88 /root/sdk/go1.20.2/src/runtime/netpoll.go:306 # 0x4c3311 internal/poll.(*pollDesc).wait+0x31 /root/sdk/go1.20.2/src/internal/poll/fd_poll_runtime.go:84 # 0x4c3ef8 internal/poll.(*pollDesc).waitRead+0x298 /root/sdk/go1.20.2/src/internal/poll/fd_poll_runtime.go:89 # 0x4c3ee6 internal/poll.(*FD).Read+0x286 /root/sdk/go1.20.2/src/internal/poll/fd_unix.go:167 # 0x519008 net.(*netFD).Read+0x28 /root/sdk/go1.20.2/src/net/fd_posix.go:55 # 0x524f64 net.(*conn).Read+0x44 /root/sdk/go1.20.2/src/net/net.go:183 # 0x67109e net/http.(*connReader).backgroundRead+0x3e /root/sdk/go1.20.2/src/net/http/server.go:674 1 @ 0x43ac56 0x4335b7 0x467369 0x4c3312 0x4c4afd 0x4c4aeb 0x51a2d5 0x52c825 0x52ba9d 0x67b3e5 0x67b01d 0x79c9af 0x79c928 0x46d021 # 0x467368 internal/poll.runtime_pollWait+0x88 /root/sdk/go1.20.2/src/runtime/netpoll.go:306 # 0x4c3311 internal/poll.(*pollDesc).wait+0x31 /root/sdk/go1.20.2/src/internal/poll/fd_poll_runtime.go:84 # 0x4c4afc internal/poll.(*pollDesc).waitRead+0x2bc /root/sdk/go1.20.2/src/internal/poll/fd_poll_runtime.go:89 # 0x4c4aea internal/poll.(*FD).Accept+0x2aa /root/sdk/go1.20.2/src/internal/poll/fd_unix.go:614 # 0x51a2d4 net.(*netFD).accept+0x34 /root/sdk/go1.20.2/src/net/fd_unix.go:172 # 0x52c824 net.(*TCPListener).accept+0x24 /root/sdk/go1.20.2/src/net/tcpsock_posix.go:148 # 0x52ba9c net.(*TCPListener).Accept+0x3c /root/sdk/go1.20.2/src/net/tcpsock.go:297 # 0x67b3e4 net/http.(*Server).Serve+0x384 /root/sdk/go1.20.2/src/net/http/server.go:3059 # 0x67b01c net/http.(*Server).ListenAndServe+0x7c /root/sdk/go1.20.2/src/net/http/server.go:2988 # 0x79c9ae net/http.ListenAndServe+0xce /root/sdk/go1.20.2/src/net/http/server.go:3242 # 0x79c927 main.cmdAdd.func1+0x47 /data/go/src/git.woa.com/bcs/bk-bcs-tencent/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/ipam/main.go:90 1 @ 0x43ac56 0x44beef 0x44bec6 0x469006 0x473985 0x6e8dbf 0x6e8d80 0x6e8b4d 0x77777a 0x77775c 0x777739 0x607c4a 0x6080d5 0x6eb0df 0x473cc2 0x6ebd57 0x6ebd1b 0x6e9a47 0x6e9e9e 0x6e8f33 0x6e8ccf 0x6ea23b 0x77786a 0x79b5f9 0x6e5ab6 0x6e5b9e 0x6e5eb1 0x6e5e37 0x79c8c9 0x43a827 0x46d021 # 0x469005 sync.runtime_SemacquireMutex+0x25 /root/sdk/go1.20.2/src/runtime/sema.go:77 # 0x473984 sync.(*Mutex).lockSlow+0x164 /root/sdk/go1.20.2/src/sync/mutex.go:171 # 0x6e8dbe sync.(*Mutex).Lock+0x7e /root/sdk/go1.20.2/src/sync/mutex.go:90 # 0x6e8d7f github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).output+0x3f /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:692 # 0x6e8b4c github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).printDepth+0xec /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:666 # 0x777779 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).print+0x79 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:657 # 0x77775b github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.Info+0x5b /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:1094 # 0x777738 github.com/Tencent/bk-bcs/bcs-common/common/blog.GlogWriter.Write+0x38 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/blog.go:33 # 0x607c49 log.(*Logger).Output+0x349 /root/sdk/go1.20.2/src/log/log.go:194 # 0x6080d4 log.Printf+0x54 /root/sdk/go1.20.2/src/log/log.go:350 # 0x6eb0de github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*logKeeper).load+0x6be /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog_file.go:185 # 0x473cc1 sync.(*Once).doSlow+0xc1 /root/sdk/go1.20.2/src/sync/once.go:74 # 0x6ebd56 sync.(*Once).Do+0x1b6 /root/sdk/go1.20.2/src/sync/once.go:65 # 0x6ebd1a github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.create+0x17a /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog_file.go:291 # 0x6e9a46 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*syncBuffer).rotateFile+0xc6 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:867 # 0x6e9e9d github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).createFiles+0xdd /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:903 # 0x6e8f32 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).output+0x1f2 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:706 # 0x6e8cce github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).printf+0x10e /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:675 # 0x6ea23a github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.Infof+0x3a /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:1112 # 0x777869 github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/ipam/types.LoadK8sArgs+0x89 /data/go/src/git.woa.com/bcs/bk-bcs-tencent/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/ipam/types/types.go:38 # 0x79b5f8 main.cmdAdd+0x178 /data/go/src/git.woa.com/bcs/bk-bcs-tencent/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/ipam/main.go:110 # 0x6e5ab5 github.com/containernetworking/cni/pkg/skel.(*dispatcher).checkVersionAndCall+0x195 /data/go/pkg/mod/github.com/containernetworking/cni@v0.6.0/pkg/skel/skel.go:162 # 0x6e5b9d github.com/containernetworking/cni/pkg/skel.(*dispatcher).pluginMain+0x7d /data/go/pkg/mod/github.com/containernetworking/cni@v0.6.0/pkg/skel/skel.go:173 # 0x6e5eb0 github.com/containernetworking/cni/pkg/skel.PluginMainWithError+0xb0 /data/go/pkg/mod/github.com/containernetworking/cni@v0.6.0/pkg/skel/skel.go:210 # 0x6e5e36 github.com/containernetworking/cni/pkg/skel.PluginMain+0x36 /data/go/pkg/mod/github.com/containernetworking/cni@v0.6.0/pkg/skel/skel.go:222 # 0x79c8c8 main.main+0xa8 /data/go/src/git.woa.com/bcs/bk-bcs-tencent/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/ipam/main.go:217 # 0x43a826 runtime.main+0x206 /root/sdk/go1.20.2/src/runtime/proc.go:250 1 @ 0x43ac56 0x44beef 0x44bec6 0x469006 0x473985 0x6ea0dd 0x6ea0ba 0x6e9fd5 0x46d021 # 0x469005 sync.runtime_SemacquireMutex+0x25 /root/sdk/go1.20.2/src/runtime/sema.go:77 # 0x473984 sync.(*Mutex).lockSlow+0x164 /root/sdk/go1.20.2/src/sync/mutex.go:171 # 0x6ea0dc sync.(*Mutex).Lock+0x3c /root/sdk/go1.20.2/src/sync/mutex.go:90 # 0x6ea0b9 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).lockAndFlushAll+0x19 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:924 # 0x6e9fd4 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).flushDaemon+0x94 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:918 1 @ 0x43ac56 0x44beef 0x44bec6 0x469006 0x473985 0x6ea0dd 0x6ea0ba 0x777625 0x777615 0x46d021 # 0x469005 sync.runtime_SemacquireMutex+0x25 /root/sdk/go1.20.2/src/runtime/sema.go:77 # 0x473984 sync.(*Mutex).lockSlow+0x164 /root/sdk/go1.20.2/src/sync/mutex.go:171 # 0x6ea0dc sync.(*Mutex).Lock+0x3c /root/sdk/go1.20.2/src/sync/mutex.go:90 # 0x6ea0b9 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.(*loggingT).lockAndFlushAll+0x19 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:924 # 0x777624 github.com/Tencent/bk-bcs/bcs-common/common/blog/glog.Flush+0x44 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/glog/glog.go:434 # 0x777614 github.com/Tencent/bk-bcs/bcs-common/common/blog.InitLogs.func1.1+0x34 /data/go/pkg/mod/github.com/!tencent/bk-bcs/bcs-common@v0.0.0-20230808080908-7c46cf1bd956/common/blog/blog.go:61 ``` TAPD: --task=074868230 --- bcs-common/common/blog/glog/glog_file.go | 7 ++-- .../bcs-netservice-controller/go.mod | 37 ++++++++++--------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/bcs-common/common/blog/glog/glog_file.go b/bcs-common/common/blog/glog/glog_file.go index c0d763d5d9..73bdc70320 100644 --- a/bcs-common/common/blog/glog/glog_file.go +++ b/bcs-common/common/blog/glog/glog_file.go @@ -35,7 +35,6 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "os/user" "path/filepath" @@ -127,7 +126,9 @@ func (lk *logKeeper) remove(tag string) (ok bool) { return } if err := lk.removeFile(block.name); err != nil { - log.Printf("remove file '%s' failed: %s", block.name, err.Error()) + // 不能使用log输出，否则会死锁问题 + // log.Printf("remove file '%s' failed: %s", block.name, err.Error()) + fmt.Printf("remove file '%s' failed: %s", block.name, err.Error()) } lk.head[tag] = block.next block = nil // for GC @@ -181,7 +182,7 @@ func (lk *logKeeper) load() { lk.total[tag]++ } else { if err = lk.removeFile(block.name); err != nil { - log.Printf("remove file '%s' failed: %s", block.name, err.Error()) + fmt.Printf("remove file '%s' failed: %s", block.name, err.Error()) } } } diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/go.mod b/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/go.mod index 454ce1c38f..4b12f811c7 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/go.mod +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-netservice-controller/go.mod @@ -3,6 +3,7 @@ module github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-netservic go 1.18 replace ( + github.com/Tencent/bk-bcs/bcs-common => ../../../../bcs-common github.com/containernetworking/cni => github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins => github.com/containernetworking/plugins v0.6.0 ) @@ -31,12 +32,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful-openapi v1.4.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -47,44 +48,44 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.7 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.41.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ugorji/go/codec v1.2.3 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect - golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.1.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.24.0 // indirect k8s.io/component-base v0.24.0 // indirect - k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect