[EXP] GTFS protobuf experiment

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/twpayne/go-geom v1.4.1
 	github.com/twpayne/go-polyline v1.1.1
-	google.golang.org/protobuf v1.33.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
@@ -47,6 +47,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect
 	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/bufbuild/protocompile v0.14.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
@@ -67,7 +68,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -54,6 +54,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 h1:aYToU0/iazkMY67/BYLt3r6/LT/m
 github.com/aws/aws-sdk-go-v2/service/sts v1.16.6/go.mod h1:rP1rEOKAGZoXp4iGDxSXFvODAtXpm34Egf0lL0eshaQ=
 github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE=
 github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
+github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw=
+github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
 github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -196,6 +198,8 @@ golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -216,6 +220,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/tlpb/gen.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/bufbuild/protocompile"
+	"github.com/interline-io/transitland-lib/tlcli"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func main() {
+	cmd := tlcli.CobraHelper(&GenGtfsCommand{}, "", "")
+	cmd.Execute()
+}
+
+type GenGtfsCommand struct {
+	Protopath string
+	Outpath   string
+	Command   *cobra.Command
+}
+
+func (cmd *GenGtfsCommand) AddFlags(fl *pflag.FlagSet) {
+}
+
+func (cmd *GenGtfsCommand) HelpDesc() (string, string) {
+	return "Generate GTFS entities", ""
+}
+
+func (cmd *GenGtfsCommand) Parse(args []string) error {
+	fl := tlcli.NewNArgs(args)
+	if fl.NArg() < 2 {
+		return errors.New("<proto> <outppath>")
+	}
+	cmd.Protopath = fl.Arg(0)
+	cmd.Outpath = fl.Arg(1)
+	return nil
+}
+
+func (cmd *GenGtfsCommand) Run() error {
+	compiler := protocompile.Compiler{
+		Resolver: &protocompile.SourceResolver{},
+	}
+	files, err := compiler.Compile(context.Background(), cmd.Protopath)
+	if err != nil {
+		return err
+	}
+	outf, err := os.Create(cmd.Outpath)
+	if err != nil {
+		return err
+	}
+	defer outf.Close()
+
+	// Go
+	outf.WriteString(`package gtfs` + "\n\n")
+	outf.WriteString(`import ( "github.com/interline-io/transitland-lib/tt" )` + "\n\n")
+
+	ttKinds := map[string]string{
+		"Url":       "tt.Url",
+		"Date":      "tt.Date",
+		"Time":      "tt.Time",
+		"Color":     "tt.Color",
+		"Key":       "tt.Key",
+		"Phone":     "tt.Phone",
+		"Email":     "tt.Email",
+		"Reference": "tt.Reference",
+		"Currency":  "tt.Currency",
+		"Language":  "tt.Language",
+		"Int":       "tt.Int",
+		"Bool":      "tt.Bool",
+		"Float":     "tt.Float",
+		"String":    "tt.String",
+		"Timezone":  "tt.Timezone",
+		"Timestamp": "tt.Timestamp",
+		"Seconds":   "tt.Seconds",
+	}
+
+	for _, lf := range files {
+		enums := lf.Enums()
+		for i := 0; i < enums.Len(); i++ {
+			en := enums.Get(i)
+			outf.WriteString(fmt.Sprintf("type %s int32\n\n", en.Name()))
+		}
+		msgs := lf.Messages()
+		for i := 0; i < msgs.Len(); i++ {
+			msg := msgs.Get(i)
+			fields := msg.Fields()
+			if _, ok := ttKinds[string(msg.Name())]; ok {
+				continue
+			}
+			if fields.Len() == 1 && fields.Get(0).Name() == "val" {
+				field := fields.Get(0)
+				outf.WriteString(fmt.Sprintf(
+					"type %s struct { tt.Option[%s] }\n\n",
+					msg.Name(),
+					mapKind(field)),
+				)
+				continue
+			}
+
+			outf.WriteString(fmt.Sprintf("type %s struct {\n", msg.Name()))
+			for j := 0; j < fields.Len(); j++ {
+				field := fields.Get(j)
+				fieldName := toCamelCase(string(field.Name()))
+				fieldKind := mapKind(field)
+				if ttKind, ok := ttKinds[fieldKind]; ok {
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, ttKind))
+					continue
+				}
+				switch fieldKind {
+				case "DatabaseEntity":
+					outf.WriteString("\tDatabaseEntity\n")
+				default:
+					outf.WriteString(fmt.Sprintf("\t%s %s\n", fieldName, fieldKind))
+				}
+			}
+			outf.WriteString("}\n\n")
+		}
+	}
+	return nil
+}
+
+func mapKind(field protoreflect.FieldDescriptor) string {
+	fieldKind := field.Kind().String()
+	switch fieldKind {
+	case "enum":
+		fieldKind = string(field.Enum().Name())
+	case "double":
+		fieldKind = "float64"
+	case "float":
+		fieldKind = "float32"
+	}
+	if fmsg := field.Message(); fmsg != nil {
+		fieldKind = string(fmsg.Name())
+	}
+	return fieldKind
+}
+
+func toCamelCase(v string) string {
+	a := strings.Split(v, "_")
+	for i := 0; i < len(a); i++ {
+		s := a[i]
+		if s == "id" {
+			s = "ID"
+		} else {
+			s = strings.ToUpper(s[0:1]) + s[1:]
+		}
+		a[i] = s
+	}
+	return strings.Join(a, "")
+}

internal/tlpb/gen_test.go

@@ -0,0 +1,135 @@
+package main
+
+// func printFirst(v []any) {
+// 	if len(v) == 0 {
+// 		return
+// 	}
+// 	fmt.Println(toJson(v[0]))
+// }
+// func printAll(v []any) {
+// 	for _, ent := range v {
+// 		fmt.Println(toJson(ent))
+// 	}
+// }
+
+// func pbJson(v protoreflect.ProtoMessage) string {
+// 	jj, _ := protojson.Marshal(v)
+// 	return string(jj)
+// }
+
+// func toJson(v any) string {
+// 	jj, _ := json.Marshal(v)
+// 	return string(jj)
+// }
+
+// var TESTFILE = ""
+// var TESTTABLE = ""
+
+// func init() {
+// 	TESTFILE = testpath.RelPath("test/data/external/bart.zip")
+// 	TESTTABLE = "stops.txt"
+// }
+
+//////////////////
+
+// func TestReadPB(t *testing.T) {
+// 	ents, err := ReadPB(TESTFILE)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	for _, ent := range ents {
+// 		fmt.Println(ent)
+// 	}
+// }
+
+// func BenchmarkReadPB(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		ReadPB(TESTFILE)
+// 	}
+// }
+
+// func ReadPB(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := &pb.Stop{}
+// 		if errs := tlcsv.LoadRow(ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadTT(t *testing.T) {
+// 	ents, err := ReadTT(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadTT(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadTT(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadTT(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }
+
+//////////////////
+
+// func TestReadG(t *testing.T) {
+// 	ents, err := ReadG(TESTFILE)
+// 	assert.NoError(t, err)
+// 	printAll(ents)
+// }
+
+// func BenchmarkReadG(b *testing.B) {
+// 	for n := 0; n < b.N; n++ {
+// 		a, _ := ReadG(TESTFILE)
+// 		_ = a
+// 		// printFirst(a)
+// 	}
+// }
+
+// func ReadG(fn string) ([]any, error) {
+// 	a := tlcsv.NewZipAdapter(fn)
+// 	if err := a.Open(); err != nil {
+// 		panic(err)
+// 	}
+// 	var ret []any
+// 	err := a.ReadRows(TESTTABLE, func(row tlcsv.Row) {
+// 		ent := gtfs.Stop{}
+// 		if errs := tlcsv.LoadRow(&ent, row); errs != nil {
+// 			for _, err := range errs {
+// 				panic(err)
+// 			}
+// 		}
+// 		ret = append(ret, ent)
+// 	})
+// 	return ret, err
+// }