✨ Initial grpc-gateway-wrapper code drop without CI

Co-authored-by: Alex Brooks <[email protected]>
Co-authored-by: Gabe Goodhart <[email protected]>
Co-authored-by: Gaurav Kumbhat <[email protected]>
Co-authored-by: Joe Runde <[email protected]>
Co-authored-by: Prashant Gupta <[email protected]>
Signed-off-by: Evaline Ju <[email protected]>

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit =
+    tests/**
+    *protobufs*

.dockerignore

@@ -0,0 +1,5 @@
+*.pyc
+__pycache__
+dist
+build
+*.egg-info

.gitignore

@@ -0,0 +1,20 @@
+working*
+build
+*.pyc
+__pycache__
+dist/
+*.egg-info
+.idea
+reports
+htmlcov
+.coverage
+.coverage.*
+
+# IDEs
+.vscode/
+.idea/
+
+# Virtual Env
+venv/
+# Mac personalization files
+*.DS_Store

.isort.cfg

@@ -0,0 +1,9 @@
+[settings]
+profile=black
+from_first=true
+import_heading_future=Future
+import_heading_stdlib=Standard
+import_heading_thirdparty=Third Party
+import_heading_firstparty=First Party
+import_heading_localfolder=Local
+known_localfolder=grpc_gateway_wrapper,tests

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.1.2
+    hooks:
+      - id: prettier
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        exclude: imports
+        additional_dependencies: ["click==8.0.4"]
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.6.3
+    hooks:
+      - id: isort
+        exclude: imports

.prettierignore

@@ -0,0 +1,2 @@
+# Don't format the swagger assets
+grpc_gateway_wrapper/resources/swagger_serve/

.whitesource

@@ -0,0 +1,3 @@
+{
+  "settingsInheritedFrom": "whitesource-config/whitesource-config@master"
+}

Dockerfile

@@ -0,0 +1,67 @@
+## deps ########################################################################
+ARG PYTHON_TAG=py39
+ARG OS=ubi8
+ARG BASE_IMAGE_TAG=latest
+ARG GO_VERSION=1.19
+ARG PROTOBUF_VERSION=3.15.8
+
+FROM golang:${GO_VERSION} as development
+
+ARG PROTOBUF_VERSION
+COPY requirements_test.txt /requirements_test.txt
+RUN true && \
+    apt-get update && \
+    apt-get install -y \
+        unzip \
+        python3.9 \
+        python3-pip && \
+    apt-get upgrade -y && \
+    pip3 install -r /requirements_test.txt && \
+    true
+
+ARG PROTOBUF_VERSION=3.15.8
+RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /protoc
+ENV PATH=${PATH}:/protoc/bin
+
+## build #######################################################################
+FROM development as build
+WORKDIR /app
+ARG PYTHON_TAG
+ARG COMPONENT_VERSION
+
+# Install twine for pushing the wheel to a pypi repository later
+RUN pip3 install twine
+
+COPY grpc_gateway_wrapper/ ./grpc_gateway_wrapper
+COPY setup.py /app/setup.py
+
+RUN python3 setup.py bdist_wheel --python-tag ${PYTHON_TAG} clean --all
+
+RUN pip3 install --no-cache-dir /app/dist/grpc_gateway_wrapper*.whl
+
+## Test ########################################################################
+FROM build as test
+COPY example /app/example
+RUN grpc-gateway-wrapper --proto_files /app/example/*.proto \
+        --metadata mm-model-id \
+        --output_dir . \
+        --install_deps
+
+## release container #####################################################################
+FROM development as release
+
+# Create a release image without any of the intermediate source files
+
+RUN true && \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    true
+
+COPY --from=build /usr/local /usr/local
+
+# Sanity check: We can import the installed wheel
+RUN grpc-gateway-wrapper --help
+ENTRYPOINT ["grpc-gateway-wrapper"]

README.md

@@ -1,2 +1,150 @@
 # grpc-gateway-wrapper
-Tool for creating a lightweight REST interface gateway to arbitrary gRPC service definitions
+The goal of this project is to generates a REST gateway wrapper layer for a grpc server with minimal customization, along with `swagger` definitions conforming to `OpenAPI 2.0`.
+
+The `gRPC-gateway`(https://github.com/grpc-ecosystem/grpc-gateway) is a plugin of the Google protocol buffers compiler protoc. It reads protobuf service definitions and generates a reverse-proxy server which translates a RESTful HTTP API into gRPC.
+
+This repo, the `grpc-gateway-wrapper` is a python library and executable that builds a `go` binary that encapsulates the reverse proxy server along with `swagger` definitions conforming to `OpenAPI 2.0`.
+
+- [gRPC Gateway Wrapper](#grpc-gateway-wrapper)
+  - [Installation instructions](#installation-instructions)
+    - [Build from source](#build-from-source)
+    - [Pull from pypi](#pull-from-pypi)
+  - [Usage](#usage)
+    - [CLI options](#cli-options)
+  - [Prerequisite](#prerequisite)
+  - [Build a Gateway](#build-a-gateway)
+    - [gRPC Metadata](#grpc-metadata)
+  - [Use a Gateway](#use-a-gateway)
+    - [Dockerized integration example](#dockerized-integration-example)
+  - [Call a Gateway](#call-a-gateway)
+  - [References](#references)
+
+## Installation instructions
+
+### Build from source
+
+```py
+python setup.py bdist_wheel
+```
+
+### Pull from pypi
+
+```py
+pip install grpc-gateway-wrapper
+```
+
+## Usage
+
+```py
+python -m grpc_gateway_wrapper
+```
+
+**or**:
+
+```sh
+grpc-gateway-wrapper
+```
+
+_(it should be installed on the system path)_.
+
+This is the main entrypoint script which can generate a working gRPC gateway server that can proxy between an equivalent REST API and a set of gRPC services.
+
+### CLI options
+
+```py
+╰$ python -m grpc_gateway_wrapper --help
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --proto_files PROTO_FILES [PROTO_FILES ...], -p PROTO_FILES [PROTO_FILES ...]
+                        The proto file to generate from
+  --working_dir WORKING_DIR, -w WORKING_DIR
+                        Location for intermediate files. If none, random will be generated
+  --no_cleanup, -c      Don't clean up working dir
+  --output_dir OUTPUT_DIR, -o OUTPUT_DIR
+                        Location for output files
+  --metadata [METADATA ...], -m [METADATA ...]
+                        gRPC metadata name(s) to add to the swagger
+  --install_deps, -d    Install go dependencies if they're missing
+  --gateway_version GATEWAY_VERSION, -g GATEWAY_VERSION
+                        Version of the grpc-gateway tools to install if installing dependencies
+  --log_level LOG_LEVEL, -l LOG_LEVEL
+                        Log level for informational logging
+```
+
+## Prerequisite
+
+1. go: https://go.dev/doc/install
+1. protoc: https://grpc.io/docs/protoc-installation/
+
+There are additional `go` dependencies which the library needs which you can directly install by passing the `--install_deps` argument.
+
+## Build a Gateway
+
+To build a gateway for your server, you need to collect the full set of `proto` files used to create the server interface, including files containing type definitions (the script is not smart enough to find them for you). With that, simply call `grpc_gateway_wrapper` with the `--protos` argument. For example:
+
+```py
+python -m grpc_gateway_wrapper \
+  --proto_files example/sample-messages.proto example/sample-service.proto
+```
+
+### gRPC Metadata
+
+Some gRPC APIs rely on the presence of certain [metadata](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) entries (e.g. [kserve model-mesh](https://github.com/kserve/modelmesh#quick-start) requires the `mm-model-id` or `mm-vmodel-id` header). These headers are not represented in the `proto` file, so they're not automatically represented in the `swagger` API generated by this template. In order to add them, you can use the `--metadata` argument to `grpc_gateway_wrapper`. For example:
+
+```py
+python -m grpc_gateway_wrapper \
+  --proto_files example/sample-messages.proto example/sample-service.proto \
+  --metadata foo bar:default_value
+```
+
+## Use a Gateway
+
+Once you build the gateway using the commands above, a `build` directory will be generated with the contents:
+
+```sh
+build
+├── app
+└── swagger
+```
+
+The generated `binary` is the `app`, along with the `swagger` directory along side it.
+
+To see the flags that can be passed to the built binary, you can run:
+
+```sh
+╰$ ./build/app --help
+
+Usage of ./build/app:
+  -mtls_client_ca="": CA certificate to use to enable mutual TLS
+  -proxy_cert="": Cert to use when making the proxy rpc call
+  -proxy_cert_hname="": Hostname override for proxy cert
+  -proxy_endpoint="localhost:8004": Endpoint for the server being proxied to
+  -proxy_no_cert_val=false: Ignore certificate validation
+  -serve_cert="": Public cert to use when serving proxy calls
+  -serve_key="": Private key to use when serving proxy calls
+  -serve_port=8080: Port to serve the gateway on
+  -swagger_path="/swagger": Absolute path to swagger assets
+```
+
+You can run it anywhere (locally, or within a deployment) and simply point it at the running `gRPC` server:
+
+```sh
+./build/app \
+  --swagger_path $PWD/build/swagger \
+  --proxy_endpoint localhost:8004 \
+  --serve_port 8080
+```
+
+(**TODO** - add TLS specific example)
+
+## Call a Gateway
+
+Once a gateway is running, you can access it directly using any HTTP client you like (`curl`, `postman`, etc...). You can also visit its swagger documentation by hitting the `/swagger` endpoint (e.g. `http://localhost:8080/swagger`).
+
+## References
+
+This project relies heavily on a few external libraries.
+
+- [`grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway): This project provides the guts of the gateway implementation, as well as the swagger spec generation
+- [`google api service`](https://cloud.google.com/endpoints/docs/grpc/grpc-service-config): In order to avoid forcing annotations into the protobuf files, this project side-loads the REST spec using the `google api service` definition.

example/sample-messages.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+package sample;
+
+/**
+ * This message is a request!
+ */
+message Request {
+    string name = 1; // (optional) The  name to request
+}
+
+/**
+ * This message is the response container with the greeting
+ */
+message Response {
+    // The greeting string
+    optional string greeting = 1;
+}

example/sample-service.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package sample.proto.v1;
+
+import "sample-messages.proto";
+
+/**
+ * This service implements a simple Hello World greeter
+ */
+service SampleService {
+    // Request a greeting for the given name
+    rpc Greeting(Request) returns (Response) {}
+}

grpc_gateway_wrapper/__init__.py

@@ -0,0 +1,5 @@
+"""
+The gRPC Gateway Wrapper project is a tool for automating the creation of a REST
+gateway server that proxies a REST API matching a gRPC API to a server that
+serves a collection of gRPC Services.
+"""

grpc_gateway_wrapper/__main__.py

@@ -0,0 +1,9 @@
+"""
+Main entrypoint for grpc_gateway_wrapper
+"""
+
+# Local
+from .gen_gateway import main
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

grpc_gateway_wrapper/add_go_package.py

@@ -0,0 +1,30 @@
+"""
+This utility function adds the "option go_package" to a proto file and resaves
+it to an output location
+"""
+
+# Local
+from .log import log
+
+
+def add_go_package(proto_file: str):
+    """
+    This utility function adds the "option go_package" to a proto file and
+    resaves it to an output location
+    """
+    log.debug("Adding go package for %s", proto_file)
+    output_lines = []
+    with open(proto_file, "r") as f_in:
+        for raw_line in f_in:
+            line = raw_line.strip()
+            if "go_package" in line:
+                log.debug("Removing original go_package [%s]", line.strip())
+            else:
+                output_lines.append(raw_line)
+            if line.startswith("package"):
+                package_name = line.split()[-1].rstrip(";").replace(".", "/")
+                output_lines.append(
+                    f'option go_package = "grpc-gateway-wrapper/{package_name}";\n'
+                )
+    with open(proto_file, "w") as f_out:
+        f_out.write("\n".join(output_lines))