Improved JSON decoding (improves arduino-cli startup time) (#1402)

* Updated library_index.json for testing * Use easyjson for parsing library_index * Added benchmarks for json deconding. Results: $ go test -v -benchmem -bench BenchmarkIndexParsing github.com/arduino/arduino-cli/arduino/libraries/librariesindex === RUN TestIndexer --- PASS: TestIndexer (0.16s) goos: linux goarch: amd64 pkg: github.com/arduino/arduino-cli/arduino/libraries/librariesindex cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz BenchmarkIndexParsingStdJSON BenchmarkIndexParsingStdJSON-8 5 214872730 ns/op 94.52 MB/s 58956539 B/op 418973 allocs/op BenchmarkIndexParsingEasyJSON BenchmarkIndexParsingEasyJSON-8 16 69215472 ns/op 293.42 MB/s 56162664 B/op 418966 allocs/op PASS ok github.com/arduino/arduino-cli/arduino/libraries/librariesindex 4.442s easyjson is 3x faster. * Updated license check cache * Added easyjson to package_index.json parser * Allow easyjson to match tags in case-insensitive See mailru/easyjson#372 * Added easyjson generation task in Taskfile * Added github action check * Update .github/workflows/check-easyjson.yml Co-authored-by: per1234 <[email protected]> Co-authored-by: per1234 <[email protected]>
arduino · May 5, 2022 · 20449fc · 20449fc
1 parent 01eab14
commit 20449fc
Show file tree

Hide file tree

Showing 21 changed files with 236,760 additions and 23,008 deletions.
diff --git a/.github/workflows/check-easyjson.yml b/.github/workflows/check-easyjson.yml
@@ -0,0 +1,56 @@
+name: Check easyjson generated files
+
+env:
+  # See: https://github.com/actions/setup-go/tree/v2#readme
+  GO_VERSION: "1.17"
+
+# See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows
+on:
+  push:
+    paths:
+      - ".github/workflows/check-easyjson.ya?ml"
+      - "go.mod"
+      - "go.sum"
+      - "Taskfile.ya?ml"
+      - "**.go"
+  pull_request:
+    paths:
+      - ".github/workflows/check-easyjson.ya?ml"
+      - "go.mod"
+      - "go.sum"
+      - "Taskfile.ya?ml"
+      - "**.go"
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  check-easyjson-generated-files:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 3.x
+
+      - name: Regenerate easyjson source code
+        run: task go:easyjson-generate
+
+      - name: Check for outdated cache
+        id: diff
+        run: |
+          git add .
+          if ! git diff --cached --color --exit-code; then
+            echo
+            echo "::error::easyjson generated files out of sync."
+            exit 1
+          fi
diff --git a/.licenses/go/github.com/josharian/intern.dep.yml b/.licenses/go/github.com/josharian/intern.dep.yml
@@ -0,0 +1,32 @@
+---
+name: github.com/josharian/intern
+version: v1.0.0
+type: go
+summary: Package intern interns strings.
+homepage: https://pkg.go.dev/github.com/josharian/intern
+license: mit
+licenses:
+- sources: license.md
+  text: |
+    MIT License
+
+    Copyright (c) 2019 Josh Bleecher Snyder
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+notices: []
diff --git a/.licenses/go/github.com/mailru/easyjson.dep.yml b/.licenses/go/github.com/mailru/easyjson.dep.yml
@@ -0,0 +1,18 @@
+---
+name: github.com/mailru/easyjson
+version: v0.7.7
+type: go
+summary: Package easyjson contains marshaler/unmarshaler interfaces and helper functions.
+homepage: https://pkg.go.dev/github.com/mailru/easyjson
+license: mit
+licenses:
+- sources: LICENSE
+  text: |
+    Copyright (c) 2016 Mail.Ru Group
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+notices: []
diff --git a/.licenses/go/github.com/mailru/easyjson/buffer.dep.yml b/.licenses/go/github.com/mailru/easyjson/buffer.dep.yml
@@ -0,0 +1,19 @@
+---
+name: github.com/mailru/easyjson/buffer
+version: v0.7.7
+type: go
+summary: Package buffer implements a buffer for serialization, consisting of a chain
+  of []byte-s to reduce copying and to allow reuse of individual chunks.
+homepage: https://pkg.go.dev/github.com/mailru/easyjson/buffer
+license: mit
+licenses:
+- sources: [email protected]/LICENSE
+  text: |
+    Copyright (c) 2016 Mail.Ru Group
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+notices: []
diff --git a/.licenses/go/github.com/mailru/easyjson/jlexer.dep.yml b/.licenses/go/github.com/mailru/easyjson/jlexer.dep.yml
@@ -0,0 +1,18 @@
+---
+name: github.com/mailru/easyjson/jlexer
+version: v0.7.7
+type: go
+summary: Package jlexer contains a JSON lexer implementation.
+homepage: https://pkg.go.dev/github.com/mailru/easyjson/jlexer
+license: mit
+licenses:
+- sources: [email protected]/LICENSE
+  text: |
+    Copyright (c) 2016 Mail.Ru Group
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+notices: []
diff --git a/.licenses/go/github.com/mailru/easyjson/jwriter.dep.yml b/.licenses/go/github.com/mailru/easyjson/jwriter.dep.yml
@@ -0,0 +1,18 @@
+---
+name: github.com/mailru/easyjson/jwriter
+version: v0.7.7
+type: go
+summary: Package jwriter contains a JSON writer.
+homepage: https://pkg.go.dev/github.com/mailru/easyjson/jwriter
+license: mit
+licenses:
+- sources: [email protected]/LICENSE
+  text: |
+    Copyright (c) 2016 Mail.Ru Group
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+notices: []
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -119,6 +119,13 @@ tasks:
     cmds:
       - go vet {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}}
 
+  go:easyjson-generate:
+    desc: Run easyjson code generation
+    cmds:
+      - go install github.com/mailru/easyjson/[email protected]
+      - easyjson arduino/cores/packageindex/index.go
+      - easyjson arduino/libraries/librariesindex/json.go
+
   # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-markdown-task/Taskfile.yml
   markdown:check-links:
     desc: Check for broken links

diff --git a/arduino/cores/packageindex/index.go b/arduino/cores/packageindex/index.go
@@ -24,17 +24,20 @@ import (
 	"github.com/arduino/arduino-cli/arduino/security"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/go-paths-helper"
+	easyjson "github.com/mailru/easyjson"
 	"github.com/sirupsen/logrus"
 	semver "go.bug.st/relaxed-semver"
 )
 
 // Index represents Cores and Tools struct as seen from package_index.json file.
+//easyjson:json
 type Index struct {
 	Packages  []*indexPackage `json:"packages"`
 	IsTrusted bool
 }
 
 // indexPackage represents a single entry from package_index.json file.
+//easyjson:json
 type indexPackage struct {
 	Name       string                  `json:"name"`
 	Maintainer string                  `json:"maintainer"`
@@ -47,6 +50,7 @@ type indexPackage struct {
 }
 
 // indexPlatformRelease represents a single Core Platform from package_index.json file.
+//easyjson:json
 type indexPlatformRelease struct {
 	Name                  string                     `json:"name"`
 	Architecture          string                     `json:"architecture"`
@@ -65,32 +69,37 @@ type indexPlatformRelease struct {
 }
 
 // indexToolDependency represents a single dependency of a core from a tool.
+//easyjson:json
 type indexToolDependency struct {
 	Packager string                 `json:"packager"`
 	Name     string                 `json:"name"`
 	Version  *semver.RelaxedVersion `json:"version"`
 }
 
 // indexDiscoveryDependency represents a single dependency of a core from a pluggable discovery tool.
+//easyjson:json
 type indexDiscoveryDependency struct {
 	Packager string `json:"packager"`
 	Name     string `json:"name"`
 }
 
 // indexMonitorDependency represents a single dependency of a core from a pluggable monitor tool.
+//easyjson:json
 type indexMonitorDependency struct {
 	Packager string `json:"packager"`
 	Name     string `json:"name"`
 }
 
 // indexToolRelease represents a single Tool from package_index.json file.
+//easyjson:json
 type indexToolRelease struct {
 	Name    string                    `json:"name"`
 	Version *semver.RelaxedVersion    `json:"version"`
 	Systems []indexToolReleaseFlavour `json:"systems"`
 }
 
 // indexToolReleaseFlavour represents a single tool flavor in the package_index.json file.
+//easyjson:json
 type indexToolReleaseFlavour struct {
 	OS              string      `json:"host"`
 	URL             string      `json:"url"`
@@ -100,17 +109,20 @@ type indexToolReleaseFlavour struct {
 }
 
 // indexBoard represents a single Board as written in package_index.json file.
+//easyjson:json
 type indexBoard struct {
 	Name string         `json:"name"`
 	ID   []indexBoardID `json:"id,omitempty"`
 }
 
 // indexBoardID represents the ID of a single board. i.e. uno, yun, diecimila, micro and the likes
+//easyjson:json
 type indexBoardID struct {
 	USB string `json:"usb"`
 }
 
 // indexHelp represents the help URL
+//easyjson:json
 type indexHelp struct {
 	Online string `json:"online,omitempty"`
 }
@@ -352,7 +364,7 @@ func LoadIndex(jsonIndexFile *paths.Path) (*Index, error) {
 		return nil, err
 	}
 	var index Index
-	err = json.Unmarshal(buff, &index)
+	err = easyjson.Unmarshal(buff, &index)
 	if err != nil {
 		return nil, err
 	}
@@ -381,7 +393,7 @@ func LoadIndexNoSign(jsonIndexFile *paths.Path) (*Index, error) {
 		return nil, err
 	}
 	var index Index
-	err = json.Unmarshal(buff, &index)
+	err = easyjson.Unmarshal(buff, &index)
 	if err != nil {
 		return nil, err
 	}