From 034739b50603cc9694a082576293e3a20f46761d Mon Sep 17 00:00:00 2001 From: freehere107 Date: Thu, 24 Jun 2021 11:48:26 +0800 Subject: [PATCH] Break change: 1. Only subscribe finalized Head 2. bump websocket connect pool and rpc lib --- configs/source/kusama.json | 5 +-- configs/source/polkadot.json | 7 ++-- go.mod | 7 ++-- go.sum | 26 ++++++++------ internal/dao/IDao.go | 3 -- internal/observer/observer.go | 2 +- internal/service/block.go | 4 +-- internal/service/chainLog.go | 3 +- internal/service/chainLog_test.go | 3 +- internal/service/event.go | 1 - internal/service/extrinsic.go | 2 -- internal/service/runtime.go | 2 ++ internal/service/subscribe_test.go | 3 ++ internal/service/substrate.go | 58 ++++++------------------------ model/substrate.go | 2 +- util/env.go | 2 +- 16 files changed, 49 insertions(+), 81 deletions(-) diff --git a/configs/source/kusama.json b/configs/source/kusama.json index ab29b05d..31f14e74 100644 --- a/configs/source/kusama.json +++ b/configs/source/kusama.json @@ -211,7 +211,7 @@ }, "Address#1050-2027": "AccountId", "Box": "BoxProposal", - "CompactAssignments#2023-?": "CompactAssignmentsLatest", + "CompactAssignments#2023-9000": "CompactAssignmentsLatest", "RefCount": "u32", "Box<::Call>": "Call", "Box<>::Proposal>": "Proposal", @@ -248,5 +248,6 @@ ] }, "ValidatorPrefs#2028-?": "ValidatorPrefsWithBlocked", - "Keys#2030-?": "SessionKeys7B" + "Keys#2030-?": "SessionKeys6", + "CompactAssignments#9010-?": "CompactAssignmentsWith24" } diff --git a/configs/source/polkadot.json b/configs/source/polkadot.json index 531e8052..e5a1236e 100644 --- a/configs/source/polkadot.json +++ b/configs/source/polkadot.json @@ -110,7 +110,7 @@ ] ] }, - "CompactAssignments#23-?": "CompactAssignmentsLatest", + "CompactAssignments#23-9000": "CompactAssignmentsLatest", "RefCount": "u32", "Box<::Call>": "Call", "Box<>::Proposal>": "Proposal", @@ -147,6 +147,7 @@ ] }, "ValidatorPrefs#28-?": "ValidatorPrefsWithBlocked", - "Keys#30-?": "SessionKeys7B", - "AccountInfo#30-?": "AccountInfoWithTripleRefCount" + "Keys#30-?": "SessionKeys6", + "AccountInfo#30-?": "AccountInfoWithTripleRefCount", + "CompactAssignments#9010-?": "CompactAssignmentsWith24" } \ No newline at end of file diff --git a/go.mod b/go.mod index 3a87b287..88f26e1b 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,8 @@ require ( github.com/golang/protobuf v1.3.5 github.com/gorilla/websocket v1.4.2 github.com/huandu/xstrings v1.3.2 - github.com/itering/scale.go v1.0.51 // indirect github.com/itering/subscan-plugin v0.2.3 - github.com/itering/substrate-api-rpc v0.3.7 + github.com/itering/substrate-api-rpc v0.4.0 github.com/jinzhu/gorm v1.9.14 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/panjf2000/ants/v2 v2.4.0 @@ -20,7 +19,7 @@ require ( github.com/shopspring/decimal v1.2.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a - golang.org/x/sys v0.0.0-20210608053332-aa57babbf139 // indirect + golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect gopkg.in/go-playground/validator.v9 v9.29.1 ) diff --git a/go.sum b/go.sum index bc741978..ece4d855 100644 --- a/go.sum +++ b/go.sum @@ -153,18 +153,16 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/itering/scale.go v0.2.4 h1:WxFfVDdy737RtVcGaC0+5KfBhkFOXFUq6T/hRznzTwg= github.com/itering/scale.go v0.2.4/go.mod h1:ufrA/8oz/qi4mtK8TXz4LFZzfJ4ZUQ+4z+cUNpY9jts= -github.com/itering/scale.go v1.0.30 h1:ThTyzp+MC1WhGTrmkZYwQ04GDAGorBbTyFU5R12NGcw= -github.com/itering/scale.go v1.0.30/go.mod h1:ufrA/8oz/qi4mtK8TXz4LFZzfJ4ZUQ+4z+cUNpY9jts= github.com/itering/scale.go v1.0.51 h1:R63gvjSAeP8sJ3ksto3U4IIaNa0XPwIr5+xyVH69Bh8= github.com/itering/scale.go v1.0.51/go.mod h1:Eg8T/07SKybkN9E1LmiEdSN/pLtV4rZDN9zuj486Gbs= github.com/itering/subscan-plugin v0.2.3 h1:8TpU/qCy+iLirkpv/4j1VXsmar7DWye8ePIFUmHG9Jk= github.com/itering/subscan-plugin v0.2.3/go.mod h1:yzh3pYoDWb37doOTaWEWxiTRsgc6mpHUAMFSwJXGmVA= github.com/itering/substrate-api-rpc v0.0.5 h1:xXmahTqKfm/gcCxG4iipSQ5FecuHNK9UjvDD68OproQ= github.com/itering/substrate-api-rpc v0.0.5/go.mod h1:Cr1wq4fy7y6ZvNB0WHJDPwCjDaE84wrmaQilC5+wFjk= -github.com/itering/substrate-api-rpc v0.3.6 h1:/KSeucgC8eeIZcByVZFEGictDM9Kw8wWbITDJ8ortb8= -github.com/itering/substrate-api-rpc v0.3.6/go.mod h1:YUzKHdUFmzoUrkMqnHiuheYqF+kNGKY89uf/ntwxCvI= -github.com/itering/substrate-api-rpc v0.3.7 h1:/iXbqq1HU0QgADcl8PlDLUkjA/kRmY3M6dI0roiEPeg= -github.com/itering/substrate-api-rpc v0.3.7/go.mod h1:t5Y0hM0lpsG+OvMV/65TNua5V/hKOu2z872ba9WwsAQ= +github.com/itering/substrate-api-rpc v0.3.8 h1:lAbF0w9EfrtVGMH6MGCEabgpzGb1CnPE07Yh01rdW7U= +github.com/itering/substrate-api-rpc v0.3.8/go.mod h1:t5Y0hM0lpsG+OvMV/65TNua5V/hKOu2z872ba9WwsAQ= +github.com/itering/substrate-api-rpc v0.4.0 h1:QFkXmQLP35W4GK+KtbArPM6UMZOW+a4iEp0qOYmPUwg= +github.com/itering/substrate-api-rpc v0.4.0/go.mod h1:t5Y0hM0lpsG+OvMV/65TNua5V/hKOu2z872ba9WwsAQ= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM= github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= @@ -195,10 +193,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= @@ -319,6 +319,7 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -326,6 +327,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= @@ -372,10 +374,10 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -437,11 +439,12 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210507014357-30e306a8bba5 h1:cez+MEm4+A0CG7ik1Qzj3bmK9DFoouuLom9lwM+Ijow= -golang.org/x/sys v0.0.0-20210507014357-30e306a8bba5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210608053332-aa57babbf139 h1:C+AwYEtBp/VQwoLntUmQ/yx3MS9vmZaKNdw5eOpoQe8= golang.org/x/sys v0.0.0-20210608053332-aa57babbf139/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -473,6 +476,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -499,6 +503,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -526,6 +531,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/dao/IDao.go b/internal/dao/IDao.go index c71873d0..300b2b31 100644 --- a/internal/dao/IDao.go +++ b/internal/dao/IDao.go @@ -28,13 +28,11 @@ type IDao interface { GetBlockList(page, row int) []model.ChainBlock BlockAsJson(c context.Context, block *model.ChainBlock) *model.ChainBlockJson CreateEvent(txn *GormDB, event *model.ChainEvent) error - DropEventNotFinalizedData(blockNum int, finalized bool) bool GetEventByBlockNum(blockNum int, where ...string) []model.ChainEventJson GetEventList(page, row int, order string, where ...string) ([]model.ChainEvent, int) GetEventsByIndex(extrinsicIndex string) []model.ChainEvent GetEventByIdx(index string) *model.ChainEvent CreateExtrinsic(c context.Context, txn *GormDB, extrinsic *model.ChainExtrinsic) error - DropExtrinsicNotFinalizedData(c context.Context, blockNum int, finalized bool) bool GetExtrinsicsByBlockNum(blockNum int) []model.ChainExtrinsicJson GetExtrinsicList(c context.Context, page, row int, order string, queryWhere ...string) ([]model.ChainExtrinsic, int) GetExtrinsicsByHash(c context.Context, hash string) *model.ChainExtrinsic @@ -42,7 +40,6 @@ type IDao interface { GetExtrinsicsDetailByIndex(c context.Context, index string) *model.ExtrinsicDetail ExtrinsicsAsJson(e *model.ChainExtrinsic) *model.ChainExtrinsicJson CreateLog(txn *GormDB, ce *model.ChainLog) error - DropLogsNotFinalizedData(blockNum int, finalized bool) bool GetLogsByIndex(index string) *model.ChainLogJson GetLogByBlockNum(blockNum int) []model.ChainLogJson SetMetadata(c context.Context, metadata map[string]interface{}) (err error) diff --git a/internal/observer/observer.go b/internal/observer/observer.go index bfdf7611..f9ba3855 100644 --- a/internal/observer/observer.go +++ b/internal/observer/observer.go @@ -91,7 +91,7 @@ LOOP: for { if dt == "substrate" { interrupt := make(chan os.Signal, 1) - subscribeConn := &recws.RecConn{KeepAliveTimeout: 10 * time.Second} + subscribeConn := &recws.RecConn{KeepAliveTimeout: 10 * time.Second, WriteTimeout: time.Second * 5, ReadTimeout: 10 * time.Second} subscribeConn.Dial(util.WSEndPoint, nil) srv.Subscribe(subscribeConn, interrupt) } else { diff --git a/internal/service/block.go b/internal/service/block.go index 938a04ab..c833dfb7 100644 --- a/internal/service/block.go +++ b/internal/service/block.go @@ -79,7 +79,7 @@ func (s *Service) CreateChainBlock(conn websocket.WsConn, hash string, block *rp if err != nil { return err } - if validator, err = s.EmitLog(txn, hash, blockNum, logs, finalized, s.ValidatorsList(conn, hash)); err != nil { + if validator, err = s.EmitLog(txn, blockNum, logs, finalized, s.ValidatorsList(conn, hash)); err != nil { return err } @@ -150,7 +150,7 @@ func (s *Service) UpdateBlockData(conn websocket.WsConn, block *model.ChainBlock return err } - validator, err := s.EmitLog(txn, block.Hash, block.BlockNum, logs, finalized, s.ValidatorsList(conn, block.Hash)) + validator, err := s.EmitLog(txn, block.BlockNum, logs, finalized, s.ValidatorsList(conn, block.Hash)) if err != nil { return err } diff --git a/internal/service/chainLog.go b/internal/service/chainLog.go index ab188531..c4f1c3a7 100644 --- a/internal/service/chainLog.go +++ b/internal/service/chainLog.go @@ -10,8 +10,7 @@ import ( "strings" ) -func (s *Service) EmitLog(txn *dao.GormDB, blockHash string, blockNum int, l []storage.DecoderLog, finalized bool, validatorList []string) (validator string, err error) { - s.dao.DropLogsNotFinalizedData(blockNum, finalized) +func (s *Service) EmitLog(txn *dao.GormDB, blockNum int, l []storage.DecoderLog, finalized bool, validatorList []string) (validator string, err error) { for index, logData := range l { dataStr := util.ToString(logData.Value) diff --git a/internal/service/chainLog_test.go b/internal/service/chainLog_test.go index 53e6aa9c..2713b091 100644 --- a/internal/service/chainLog_test.go +++ b/internal/service/chainLog_test.go @@ -10,7 +10,6 @@ func TestService_EmitLog(t *testing.T) { txn := testSrv.dao.DbBegin() defer testSrv.dao.DbRollback(txn) - blockHash := "0x5f3d69b39b3634913965bdedea920f9156eaed992851f30fd6fc19d798ac764b" logs, err := substrate.DecodeLogDigest([]string{ "0x0642414245b5010102000000efa6cd0f000000004618a29aeb02e8ae7bb2360d8f5f13828c3c2f9fd15bc674be6e2c64be17a00ebb8fa2449c7b19b5988d6110e0f03a44693f246597e7bdf1a4b48aa4c50b600e6252c08951731c00e11a7f5a6b26d7c6bdf421145c575a03c23420bd76decd06", "0x00904d4d5252aec4a1a273aca92e65330af40d9b06447427454910e0e1b9fc9e2157b670a30f", @@ -18,7 +17,7 @@ func TestService_EmitLog(t *testing.T) { }) assert.NoError(t, err) validatorList := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10"} - validator, err := testSrv.EmitLog(txn, blockHash, 300000, logs, true, validatorList) + validator, err := testSrv.EmitLog(txn, 300000, logs, true, validatorList) assert.NoError(t, err) assert.Equal(t, "v3", validator) diff --git a/internal/service/event.go b/internal/service/event.go index c0205ae6..1d8a1397 100644 --- a/internal/service/event.go +++ b/internal/service/event.go @@ -16,7 +16,6 @@ func (s *Service) AddEvent( hashMap map[string]string, feeMap map[string]decimal.Decimal) (eventCount int, err error) { - s.dao.DropEventNotFinalizedData(block.BlockNum, block.Finalized) for _, event := range e { event.ModuleId = strings.ToLower(event.ModuleId) event.ExtrinsicHash = hashMap[fmt.Sprintf("%d-%d", block.BlockNum, event.ExtrinsicIdx)] diff --git a/internal/service/extrinsic.go b/internal/service/extrinsic.go index 79fa5882..6ef8e08d 100644 --- a/internal/service/extrinsic.go +++ b/internal/service/extrinsic.go @@ -32,8 +32,6 @@ func (s *Service) createExtrinsic(c context.Context, hash := make(map[string]string) - s.dao.DropExtrinsicNotFinalizedData(c, block.BlockNum, block.Finalized) - for index, extrinsic := range e { extrinsic.CallModule = strings.ToLower(extrinsic.CallModule) extrinsic.BlockNum = block.BlockNum diff --git a/internal/service/runtime.go b/internal/service/runtime.go index e0ad37b8..e4609215 100644 --- a/internal/service/runtime.go +++ b/internal/service/runtime.go @@ -35,6 +35,8 @@ func (s *Service) regRuntimeVersion(name string, spec int, hash ...string) error if coded := s.regCodecMetadata(hash...); coded != "" { runtime := metadata.RegNewMetadataType(spec, coded) s.setRuntimeData(spec, runtime, coded) + } else { + panic("get runtime metadata error") } } runtimeSpecs = append(runtimeSpecs, spec) diff --git a/internal/service/subscribe_test.go b/internal/service/subscribe_test.go index 22c306f1..2afdc727 100644 --- a/internal/service/subscribe_test.go +++ b/internal/service/subscribe_test.go @@ -29,6 +29,9 @@ type TestConn struct { writer *Buffer // the current writer returned to the application } +func (t *TestConn) CloseAndReconnect() { +} + func (t *TestConn) MarkUnusable() {} func (t *TestConn) Dial(urlStr string, reqHeader http.Header) { diff --git a/internal/service/substrate.go b/internal/service/substrate.go index b4f1db72..2ae01953 100644 --- a/internal/service/substrate.go +++ b/internal/service/substrate.go @@ -31,8 +31,8 @@ type subscription struct { } var ( - onceNewHead, onceFinHead sync.Once - subscriptionIds = []subscription{{Topic: ChainNewHead}, {Topic: ChainFinalizedHead}, {Topic: StateStorage}} + onceFinHead sync.Once + subscriptionIds = []subscription{{Topic: ChainNewHead}, {Topic: ChainFinalizedHead}, {Topic: StateStorage}} ) type SubscribeService struct { @@ -79,12 +79,6 @@ func (s *SubscribeService) parser(message []byte) (err error) { r := j.ToNewHead() _ = s.updateChainMetadata(map[string]interface{}{"blockNum": util.HexToNumStr(r.Number)}) upgradeHealth(j.Method) - go func() { - s.newHead <- true - onceNewHead.Do(func() { - go s.subscribeFetchBlock() - }) - }() case ChainFinalizedHead: r := j.ToNewHead() _ = s.updateChainMetadata(map[string]interface{}{"finalized_blockNum": util.HexToNumStr(r.Number)}) @@ -126,28 +120,6 @@ func (s *SubscribeService) subscribeFetchBlock() { defer p.Release() for { select { - case <-s.newHead: - best, err := s.dao.GetBestBlockNum(ctx) - if err != nil || best == 0 { - time.Sleep(BlockTime * time.Second) - return - } - lastNum, _ := s.dao.GetFillBestBlockNum(ctx) - finalizedBlock, _ := s.dao.GetFinalizedBlockNum(ctx) - - startBlock := lastNum + 1 - if lastNum == 0 { - startBlock = lastNum - } - - for i := startBlock; i <= int(best); i++ { - wg.Add(1) - _ = p.Invoke(BlockFinalized{ - BlockNum: i, - Finalized: finalizedBlock >= FinalizedWaitingBlockCount && uint64(i) <= finalizedBlock-FinalizedWaitingBlockCount, - }) - } - wg.Wait() case <-s.newFinHead: final, err := s.dao.GetFinalizedBlockNum(context.TODO()) if err != nil || final == 0 { @@ -238,23 +210,15 @@ func (s *Service) FillBlockData(conn websocket.WsConn, blockNum int, finalized b } // refresh finalized info for update if block != nil { - // Confirm data, only set block Finalized - if block.Hash == blockHash && block.ExtrinsicsRoot == rpcBlock.Block.Header.ExtrinsicsRoot && block.Event == event && !block.CodecError && finalized { - s.dao.SetBlockFinalized(block) - } else { - // refresh all block data - block.ExtrinsicsRoot = rpcBlock.Block.Header.ExtrinsicsRoot - block.Hash = blockHash - block.ParentHash = rpcBlock.Block.Header.ParentHash - block.StateRoot = rpcBlock.Block.Header.StateRoot - - block.Extrinsics = util.ToString(rpcBlock.Block.Extrinsics) - block.Logs = util.ToString(rpcBlock.Block.Header.Digest.Logs) - block.Event = event - - _ = s.UpdateBlockData(conn, block, finalized) - } - setFinalized() + // Confirm data, only set block Finalized, refresh all block data + block.ExtrinsicsRoot = rpcBlock.Block.Header.ExtrinsicsRoot + block.Hash = blockHash + block.ParentHash = rpcBlock.Block.Header.ParentHash + block.StateRoot = rpcBlock.Block.Header.StateRoot + block.Extrinsics = util.ToString(rpcBlock.Block.Extrinsics) + block.Logs = util.ToString(rpcBlock.Block.Header.Digest.Logs) + block.Event = event + _ = s.UpdateBlockData(conn, block, finalized) return } // for Create diff --git a/model/substrate.go b/model/substrate.go index 70562d5d..14a00627 100644 --- a/model/substrate.go +++ b/model/substrate.go @@ -128,7 +128,7 @@ type RuntimeVersion struct { Id int `json:"-"` Name string `json:"-"` SpecVersion int `json:"spec_version"` - Modules string `json:"modules"` + Modules string `json:"modules" sql:"type:TEXT;"` RawData string `json:"-" sql:"type:MEDIUMTEXT;"` } diff --git a/util/env.go b/util/env.go index ee9b3526..6288ab98 100644 --- a/util/env.go +++ b/util/env.go @@ -10,7 +10,7 @@ var ( AddressType = GetEnv("SUBSTRATE_ADDRESS_TYPE", "1") BalanceAccuracy = GetEnv("SUBSTRATE_ACCURACY", "9") CommissionAccuracy = GetEnv("COMMISSION_ACCURACY", "9") - WSEndPoint = GetEnv("CHAIN_WS_ENDPOINT", "wss://rpc.polkadot.io") + WSEndPoint = GetEnv("CHAIN_WS_ENDPOINT", "wss://polkadot.elara.patract.io") NetworkNode = GetEnv("NETWORK_NODE", "polkadot") IsProduction = os.Getenv("DEPLOY_ENV") == "prod" )