Create IDE query script

This will be the integration point to provide build artifacts to Cider G.

NOTE FOR REVIEWERS - original patch and result patch are not identical.
PLEASE REVIEW CAREFULLY.
Diffs between the patches:
 	files := flag.Args()
> -
> -			if prev, ok := modules[f]; ok && !strings.HasSuffix(prev.Name, ".impl") {
> -				log.Printf("File %q found in module %q but is already part of module %q", f, m.Name, prev.Name)
> +			if modules[f] != nil {
> +				log.Printf("File %q found in module %q but is already covered by module %q", f, m.Name, modules[f].Name)
> -		var genFiles []*pb.GeneratedFile
> +		var generated []*pb.GeneratedFile
> -				// Note: Contents will be filled below.
> -				genFiles = append(genFiles, &pb.GeneratedFile{Path: relPath})
> +				contents, err := os.ReadFile(d)
> +				if err != nil {
> +					fmt.Printf("Generated file %q not found - will be skipped.\n", d)
> +					continue
> +				}
> +
> +				generated = append(generated, &pb.GeneratedFile{
> +					Path:     relPath,
> +					Contents: contents,
> +				})
> -		file.Generated = genFiles
> +		file.Generated = generated
> -	for _, s := range sources {
> -		for _, g := range s.GetGenerated() {
> -			contents, err := os.ReadFile(path.Join(env.OutDir, g.GetPath()))
> -			if err != nil {
> -				fmt.Printf("Failed to read generated file %q: %v. File contents will be missing.\n", g.GetPath(), err)
> -				continue
> -			}
> -			g.Contents = contents
> -		}
> -	}
> -
> -		if strings.HasSuffix(name, "-jarjar") {
> +		if strings.HasSuffix(name, "-jarjar") || strings.HasSuffix(name, ".impl") {

Original patch:
 diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
old mode 100644
new mode 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -1,3 +1,5 @@
+// Binary ide_query generates and analyzes build artifacts.
+// The produced result can be consumed by IDEs to provide language features.
 package main

 import (
@@ -34,10 +36,10 @@

 var _ flag.Value = (*LunchTarget)(nil)

-// Get implements flag.Value.
-func (l *LunchTarget) Get() any {
-	return l
-}
+// // Get implements flag.Value.
+// func (l *LunchTarget) Get() any {
+// 	return l
+// }

 // Set implements flag.Value.
 func (l *LunchTarget) Set(s string) error {
@@ -64,13 +66,12 @@
 	env.RepoDir = os.Getenv("TOP")
 	flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query")
 	flag.Parse()
-	if flag.NArg() == 0 {
+	files := flag.Args()
+	if len(files) == 0 {
 		fmt.Println("No files provided.")
 		os.Exit(1)
 		return
 	}
-
-	files := flag.Args()

 	ctx := context.Background()
 	javaDepsPath := pa
[[[Original patch trimmed due to size. Decoded string size: 2916. Decoded string SHA1: 5d8fd4a92cc403da51c9ddb8442da2e391e6fcb1.]]]

Result patch:
 diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
index 2e76738..0fdb6de 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -1,3 +1,5 @@
+// Binary ide_query generates and analyzes build artifacts.
+// The produced result can be consumed by IDEs to provide language features.
 package main

 import (
@@ -34,10 +36,10 @@

 var _ flag.Value = (*LunchTarget)(nil)

-// Get implements flag.Value.
-func (l *LunchTarget) Get() any {
-	return l
-}
+// // Get implements flag.Value.
+// func (l *LunchTarget) Get() any {
+// 	return l
+// }

 // Set implements flag.Value.
 func (l *LunchTarget) Set(s string) error {
@@ -64,14 +66,13 @@
 	env.RepoDir = os.Getenv("TOP")
 	flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query")
 	flag.Parse()
-	if flag.NArg() == 0 {
+	files := flag.Args()
+	if len(files) == 0 {
 		fmt.Println("No files provided.")
 		os.Exit(1)
 		return
 	}

-	files := flag.Args()
-
 	ctx := context.Background()
 	javaDepsPath := path
[[[Result patch trimmed due to size. Decoded string size: 3022. Decoded string SHA1: a8824749eafbbb8d09c4e95fe491a16e3ea82569.]]]

NOTE FOR REVIEWERS - original patch and result patch are not identical.
PLEASE REVIEW CAREFULLY.
Diffs between the patches:
 	var javaFiles []string
> +	for _, f := range files {
> +		switch {
> +		case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
> +			javaFiles = append(javaFiles, f)
> +		default:
> +			log.Printf("File %q is supported - will be skipped.", f)
> +		}
> +	}
> +
> -	modules := make(map[string]*javaModule) // file path -> module
> -	for _, f := range files {
> +	fileToModule := make(map[string]*javaModule) // file path -> module
> +	for _, f := range javaFiles {
> -			if modules[f] != nil {
> -				log.Printf("File %q found in module %q but is already covered by module %q", f, m.Name, modules[f].Name)
> +			if fileToModule[f] != nil {
> +				// TODO(michaelmerg): Handle the case where a file is covered by multiple modules.
> +				log.Printf("File %q found in module %q but is already covered by module %q", f, m.Name, fileToModule[f].Name)
> -			modules[f] = m
> +			fileToModule[f] = m
> -	for _, m := range modules {
> +	for _, m := range fileToModule {
> +	type depsAndGenerated struct {
> +		Deps      []string
> +		Generated []*pb.GeneratedFile
> +	}
> +	moduleToDeps := make(map[string]*depsAndGenerated)
> -		m := modules[f]
> +		m := fileToModule[f]
> +		file.Status = &pb.Status{Code: pb.Status_OK}
> +		if moduleToDeps[m.Name] != nil {
> +			file.Generated = moduleToDeps[m.Name].Generated
> +			file.Deps = moduleToDeps[m.Name].Deps
> +			continue
> +		}
> +
> -
> +		moduleToDeps[m.Name] = &depsAndGenerated{deps, generated}
> -		file.Status = &pb.Status{Code: pb.Status_OK}

Original patch:
 diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
old mode 100644
new mode 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -72,6 +72,16 @@
 		os.Exit(1)
 		return
 	}
+
+	var javaFiles []string
+	for _, f := range files {
+		switch {
+		case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
+			javaFiles = append(javaFiles, f)
+		default:
+			log.Printf("File %q is supported - will be skipped.", f)
+		}
+	}

 	ctx := context.Background()
 	javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json")
@@ -85,22 +95,23 @@
 		log.Fatalf("Failed to load java modules: %v", err)
 	}

-	modules := make(map[string]*javaModule) // file path -> module
-	for _, f := range files {
+	fileToModule := make(map[string]*javaModule) // file path -> module
+	for _, f := range javaFiles {
 		for _, m := range javaModules {
 			if !slices.Contains(m.Srcs, f) {
 				continue
 			}
-			if modules[f] != nil {
-				log.Printf("File %q found in
[[[Original patch trimmed due to size. Decoded string size: 2629. Decoded string SHA1: 4517ba713fdb898ba9d77c4acbe934c08a2d9fe0.]]]

Result patch:
 diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
index 0fdb6de..7335875 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -73,6 +73,16 @@
 		return
 	}

+	var javaFiles []string
+	for _, f := range files {
+		switch {
+		case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
+			javaFiles = append(javaFiles, f)
+		default:
+			log.Printf("File %q is supported - will be skipped.", f)
+		}
+	}
+
 	ctx := context.Background()
 	javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json")
 	// TODO(michaelmerg): Figure out if module_bp_java_deps.json is outdated.
@@ -85,22 +95,23 @@
 		log.Fatalf("Failed to load java modules: %v", err)
 	}

-	modules := make(map[string]*javaModule) // file path -> module
-	for _, f := range files {
+	fileToModule := make(map[string]*javaModule) // file path -> module
+	for _, f := range javaFiles {
 		for _, m := range javaModules {
 			if !slices.Contains(m.Srcs, f) {
 				continue
 			}

[[[Result patch trimmed due to size. Decoded string size: 2717. Decoded string SHA1: 5e5223251ebdc548258bc27daf3528d662c39410.]]]

Change-Id: Ibe5d386399affd2951206bb5a714972e0e2fee92

tools/ide_query/go.mod

@@ -0,0 +1,7 @@
+module ide_query
+
+go 1.21
+
+require (
+  	google.golang.org/protobuf v0.0.0
+)

tools/ide_query/go.work

@@ -0,0 +1,9 @@
+go 1.21
+
+use (
+	.
+)
+
+replace (
+	google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf
+)

tools/ide_query/go.work.sum

@@ -0,0 +1,5 @@
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=

tools/ide_query/ide_query.go

@@ -0,0 +1,265 @@
+// Binary ide_query generates and analyzes build artifacts.
+// The produced result can be consumed by IDEs to provide language features.
+package main
+
+import (
+	"container/list"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"slices"
+	"strings"
+
+	"google.golang.org/protobuf/proto"
+	pb "ide_query/ide_query_proto"
+)
+
+// Env contains information about the current environment.
+type Env struct {
+	LunchTarget LunchTarget
+	RepoDir     string
+	OutDir      string
+}
+
+// LunchTarget is a parsed Android lunch target.
+// Input format: <product_name>-<release_type>-<build_variant>
+type LunchTarget struct {
+	Product string
+	Release string
+	Variant string
+}
+
+var _ flag.Value = (*LunchTarget)(nil)
+
+// // Get implements flag.Value.
+// func (l *LunchTarget) Get() any {
+// 	return l
+// }
+
+// Set implements flag.Value.
+func (l *LunchTarget) Set(s string) error {
+	parts := strings.Split(s, "-")
+	if len(parts) != 3 {
+		return fmt.Errorf("invalid lunch target: %q, must have form <product_name>-<release_type>-<build_variant>", s)
+	}
+	*l = LunchTarget{
+		Product: parts[0],
+		Release: parts[1],
+		Variant: parts[2],
+	}
+	return nil
+}
+
+// String implements flag.Value.
+func (l *LunchTarget) String() string {
+	return fmt.Sprintf("%s-%s-%s", l.Product, l.Release, l.Variant)
+}
+
+func main() {
+	var env Env
+	env.OutDir = os.Getenv("OUT_DIR")
+	env.RepoDir = os.Getenv("ANDROID_BUILD_TOP")
+	flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query")
+	flag.Parse()
+	files := flag.Args()
+	if len(files) == 0 {
+		fmt.Println("No files provided.")
+		os.Exit(1)
+		return
+	}
+
+	var javaFiles []string
+	for _, f := range files {
+		switch {
+		case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
+			javaFiles = append(javaFiles, f)
+		default:
+			log.Printf("File %q is supported - will be skipped.", f)
+		}
+	}
+
+	ctx := context.Background()
+	javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json")
+	// TODO(michaelmerg): Figure out if module_bp_java_deps.json is outdated.
+	runMake(ctx, env, "nothing")
+
+	javaModules, err := loadJavaModules(javaDepsPath)
+	if err != nil {
+		log.Fatalf("Failed to load java modules: %v", err)
+	}
+
+	fileToModule := make(map[string]*javaModule) // file path -> module
+	for _, f := range javaFiles {
+		for _, m := range javaModules {
+			if !slices.Contains(m.Srcs, f) {
+				continue
+			}
+			if fileToModule[f] != nil {
+				// TODO(michaelmerg): Handle the case where a file is covered by multiple modules.
+				log.Printf("File %q found in module %q but is already covered by module %q", f, m.Name, fileToModule[f].Name)
+				continue
+			}
+			fileToModule[f] = m
+		}
+	}
+
+	var toMake []string
+	for _, m := range fileToModule {
+		toMake = append(toMake, m.Name)
+	}
+	fmt.Printf("Running make for modules: %v\n", strings.Join(toMake, ", "))
+	if err := runMake(ctx, env, toMake...); err != nil {
+		log.Fatalf("Failed to run make: %v", err)
+	}
+
+	var sources []*pb.SourceFile
+	type depsAndGenerated struct {
+		Deps      []string
+		Generated []*pb.GeneratedFile
+	}
+	moduleToDeps := make(map[string]*depsAndGenerated)
+	for _, f := range files {
+		file := &pb.SourceFile{
+			Path:       f,
+			WorkingDir: env.RepoDir,
+		}
+		sources = append(sources, file)
+
+		m := fileToModule[f]
+		if m == nil {
+			file.Status = &pb.Status{
+				Code:    pb.Status_FAILURE,
+				Message: proto.String("File not found in any module."),
+			}
+			continue
+		}
+
+		file.Status = &pb.Status{Code: pb.Status_OK}
+		if moduleToDeps[m.Name] != nil {
+			file.Generated = moduleToDeps[m.Name].Generated
+			file.Deps = moduleToDeps[m.Name].Deps
+			continue
+		}
+
+		deps := transitiveDeps(m, javaModules)
+		var generated []*pb.GeneratedFile
+		outPrefix := env.OutDir + "/"
+		for _, d := range deps {
+			if relPath, ok := strings.CutPrefix(d, outPrefix); ok {
+				contents, err := os.ReadFile(d)
+				if err != nil {
+					fmt.Printf("Generated file %q not found - will be skipped.\n", d)
+					continue
+				}
+
+				generated = append(generated, &pb.GeneratedFile{
+					Path:     relPath,
+					Contents: contents,
+				})
+			}
+		}
+		moduleToDeps[m.Name] = &depsAndGenerated{deps, generated}
+		file.Generated = generated
+		file.Deps = deps
+	}
+
+	res := &pb.IdeAnalysis{
+		BuildArtifactRoot: env.OutDir,
+		Sources:           sources,
+		Status:            &pb.Status{Code: pb.Status_OK},
+	}
+	data, err := proto.Marshal(res)
+	if err != nil {
+		log.Fatalf("Failed to marshal result proto: %v", err)
+	}
+
+	err = os.WriteFile(path.Join(env.OutDir, "ide_query.pb"), data, 0644)
+	if err != nil {
+		log.Fatalf("Failed to write result proto: %v", err)
+	}
+
+	for _, s := range sources {
+		fmt.Printf("%s: %v (Deps: %d, Generated: %d)\n", s.GetPath(), s.GetStatus(), len(s.GetDeps()), len(s.GetGenerated()))
+	}
+}
+
+// runMake runs Soong build for the given modules.
+func runMake(ctx context.Context, env Env, modules ...string) error {
+	args := []string{
+		"--make-mode",
+		"ANDROID_BUILD_ENVIRONMENT_CONFIG=googler-cog",
+		"TARGET_PRODUCT=" + env.LunchTarget.Product,
+		"TARGET_RELEASE=" + env.LunchTarget.Release,
+		"TARGET_BUILD_VARIANT=" + env.LunchTarget.Variant,
+	}
+	args = append(args, modules...)
+	cmd := exec.CommandContext(ctx, "build/soong/soong_ui.bash", args...)
+	cmd.Dir = env.RepoDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+type javaModule struct {
+	Name    string
+	Path    []string `json:"path,omitempty"`
+	Deps    []string `json:"dependencies,omitempty"`
+	Srcs    []string `json:"srcs,omitempty"`
+	Jars    []string `json:"jars,omitempty"`
+	SrcJars []string `json:"srcjars,omitempty"`
+}
+
+func loadJavaModules(path string) (map[string]*javaModule, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var ret map[string]*javaModule // module name -> module
+	if err = json.Unmarshal(data, &ret); err != nil {
+		return nil, err
+	}
+
+	for name, module := range ret {
+		if strings.HasSuffix(name, "-jarjar") || strings.HasSuffix(name, ".impl") {
+			delete(ret, name)
+			continue
+		}
+
+		module.Name = name
+	}
+	return ret, nil
+}
+
+func transitiveDeps(m *javaModule, modules map[string]*javaModule) []string {
+	var ret []string
+	q := list.New()
+	q.PushBack(m.Name)
+	seen := make(map[string]bool) // module names -> true
+	for q.Len() > 0 {
+		name := q.Remove(q.Front()).(string)
+		mod := modules[name]
+		if mod == nil {
+			continue
+		}
+
+		ret = append(ret, mod.Srcs...)
+		ret = append(ret, mod.SrcJars...)
+		ret = append(ret, mod.Jars...)
+		for _, d := range mod.Deps {
+			if seen[d] {
+				continue
+			}
+			seen[d] = true
+			q.PushBack(d)
+		}
+	}
+	slices.Sort(ret)
+	ret = slices.Compact(ret)
+	return ret
+}

tools/ide_query/ide_query.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+cd $(dirname $BASH_SOURCE)
+source $(pwd)/../../shell_utils.sh
+require_top
+
+# Ensure cogsetup (out/ will be symlink outside the repo)
+. ${TOP}/build/make/cogsetup.sh
+
+export ANDROID_BUILD_TOP=$TOP
+export OUT_DIR=${OUT_DIR}
+exec "${TOP}/prebuilts/go/linux-x86/bin/go" "run" "ide_query" "$@"