feat: Update endpoint deserializer

The endpoint has changed the message serializer to flatbuffer.
This commit add the build script for generate endpoint flatbuffer
deserializer and deserialize the endpoint mam message from IOTA network.

.github/workflows/go.yml

@@ -10,7 +10,7 @@ jobs:
 
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04 
     steps:
 
     - name: Set up Go 1.x
@@ -28,9 +28,11 @@ jobs:
             curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
             dep ensure
         fi
+    - name: Install dependencies 
+      run: sudo apt-get install flatbuffers-compiler libbrotli-dev
 
     - name: Build
-      run: go build -v ./...
+      run: ./build.sh 
 
     - name: Test
       run: go test -v ./test/...

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/vehicle-scheme"]
+	path = third_party/vehicle-scheme
+	url = https://github.com/DLTcollab/vehicle-scheme.git

models/obd/obd.go → OBD/OBD2_json.go

@@ -1,6 +1,6 @@
-package obd
+package OBD
 
-type ODB2_data struct {
+type OBD2_json struct {
 	Vin                                 string  `json:"vin"`
 	Engine_load                         int     `json:"engine_load"`
 	Engine_coolant_temperature          int     `json:"engine_coolant_temperature"`
@@ -26,37 +26,27 @@ type ODB2_data struct {
 	High_beam_head_light                int     `json:"high_beam_head_light"`
 }
 
-type OBD struct {
-	PID uint8
-}
-
-// Service 09
-// PID:0x02
-func get_vechicle_identification_number(data string) string {
-	return data
-}
-
 // Service 01
 // PID:0x04
-func get_calculated_engine_load(data uint32) int {
+func GetCalculatedEngineLoad(data uint32) int {
 	var A = data >> 24
 	return int(A) * 100 / 255
 }
 
 // PID:0x05
-func get_engine_coolant_temperature(data uint32) int {
+func GetEngineCoolantTemperature(data uint32) int {
 	var A = data >> 24
 	return int(A) - 40
 }
 
 // PID:0x0A
-func get_fuel_pressure(data uint32) int {
+func GetFuelPressure(data uint32) int {
 	var A = data >> 24
 	return int(A) * 3
 }
 
 // PID:0x0C
-func get_engine_speed(data uint32) float32 {
+func GetEngineSpeed(data uint32) float32 {
 	var A = data >> 24
 	var B = data << 8
 	B = B >> 24
@@ -65,19 +55,19 @@ func get_engine_speed(data uint32) float32 {
 }
 
 // PID:0x0D
-func get_vehicle_speed(data uint32) int {
+func GetVehicleSpeed(data uint32) int {
 	var A = data >> 24
 	return int(A)
 }
 
 // PID:0x0F
-func get_intake_air_temperature(data uint32) int {
+func GetIntakeAirTemperature(data uint32) int {
 	var A = data >> 24
 	return int(A) - 40
 }
 
 // PID:0x10
-func get_mass_air_flow(data uint32) int {
+func GetMassAirFlow(data uint32) int {
 	var A = data >> 24
 	var B = data << 8
 	B = B >> 24
@@ -86,19 +76,19 @@ func get_mass_air_flow(data uint32) int {
 }
 
 // PID:0x2F
-func get_fuel_tank_level_input(data uint32) int {
+func GetFuelTankLevelInput(data uint32) int {
 	var A = data >> 24
 	return int(A) * 100 / 255
 }
 
 // PID:0x33
-func get_absolute_barometric_pressure(data uint32) int {
+func GetAbsoluteBarometricPressure(data uint32) int {
 	var A = data >> 24
 	return int(A)
 }
 
 // PID:0x42
-func get_control_module_voltage(data uint32) float32 {
+func GetControlModuleVoltage(data uint32) float32 {
 	var A = data >> 24
 	var B = data << 8
 	B = B >> 24
@@ -107,69 +97,33 @@ func get_control_module_voltage(data uint32) float32 {
 }
 
 // PID:0x45
-func get_throttle_position(data uint32) int {
+func GetThrottlePosition(data uint32) int {
 	var A = data >> 24
 	return int(A) * 100 / 255
 }
 
 // PID:0x46
-func get_ambient_air_temperature(data uint32) int {
+func GetAmbientAirTemperature(data uint32) int {
 	var A = data >> 24
 	return int(A) - 40
 }
 
 // PID:0x5A
-func get_relative_accelerator_pedal_position(data uint32) int {
+func GetRelativeAcceleratorPedalPosition(data uint32) int {
 	var A = data >> 24
 	return int(A) * 100 / 255
 }
 
 // PID:0x5C
-func get_engine_oil_temperature(data uint32) int {
+func GetEngineOilTemperature(data uint32) int {
 	var A = data >> 24
 	return int(A) - 40
 }
 
 // PID:0x5E
-func get_engine_fuel_rate(data uint32) float32 {
+func GetEngineFuelRate(data uint32) float32 {
 	var A = data >> 24
 	var B = data << 8
 	B = B >> 24
 	return (float32(A)*256 + float32(B)) / 20
 }
-
-// E1~E8 is not inside obd2-pids standard
-// PID:E1
-func get_service_distance(data uint32) uint32 {
-	return data
-}
-
-// PID:E2
-func get_anti_lock_barking_active(data uint32) uint32 {
-	return data
-}
-
-// PID:E3
-func get_steering_wheel_angle(data uint32) uint32 {
-	return data
-}
-
-// PID:E5
-func get_position_of_doors(data uint32) uint32 {
-	return data
-}
-
-// PID:E6
-func get_right_left_turn_signal_light(data uint32) uint32 {
-	return data
-}
-
-// PID:E7
-func get_alternate_beam_head_light(data uint32) uint32 {
-	return data
-}
-
-// PID:E8
-func get_high_beam_head_light(data uint32) uint32 {
-	return data
-}

build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+git submodule update --init --recursive
+flatc --go third_party/vehicle-scheme/*.fbs
+go build -v ./...

controllers/controller.go

@@ -2,20 +2,19 @@ package controller
 
 import (
 	"bytes"
-	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"strings"
 
+	"github.com/DLTcollab/vehicle-data-explorer/OBD"
 	"github.com/DLTcollab/vehicle-data-explorer/models/device"
 	"github.com/DLTcollab/vehicle-data-explorer/models/elasticsearch"
 	"github.com/DLTcollab/vehicle-data-explorer/models/endpoint_CBCDecrypter"
 	"github.com/DLTcollab/vehicle-data-explorer/models/endpoint_deserializer"
 	"github.com/DLTcollab/vehicle-data-explorer/models/jwt"
-	"github.com/DLTcollab/vehicle-data-explorer/models/obd"
+	"github.com/google/brotli/go/cbrotli"
 
 	"github.com/gin-gonic/gin"
 )
@@ -34,7 +33,7 @@ type MAM_response struct {
 type Endpoint_obd2_data struct {
 	Timestamp int64         `json:"timestamp"`
 	Device_id string        `json"device_id"`
-	Data      obd.ODB2_data `json:"data"`
+	Data      OBD.OBD2_json `json:"data"`
 }
 
 type MAM_post_data struct {
@@ -135,8 +134,40 @@ func Get_dashboard_realtime_data(c *gin.Context) {
 
 	for i := 0; i < len(mam_response.Payload); i++ {
 		payload := mam_response.Payload[i]
-		endpoint_data := Descrypt_mam_response(payload, private_key)
-		data = append(data, endpoint_data)
+		obd2Meta := Descrypt_mam_response(payload, private_key)
+		obd2Data := new(OBD.OBD2_data)
+		obd2Meta.Data(obd2Data)
+
+		endpointData := Endpoint_obd2_data{
+			Timestamp: obd2Meta.Timestamp(),
+			Device_id: string(obd2Meta.DeviceID()),
+			Data: OBD.OBD2_json{
+				Vin:                                 string(obd2Data.Vin()),
+				Engine_load:                         OBD.GetCalculatedEngineLoad(obd2Data.EngineLoad()),
+				Engine_coolant_temperature:          OBD.GetEngineCoolantTemperature(obd2Data.EngineCoolantTemperature()),
+				Fuel_pressure:                       OBD.GetFuelPressure(obd2Data.FuelPressure()),
+				Engine_speed:                        OBD.GetEngineSpeed(obd2Data.EngineSpeed()),
+				Vehicle_speed:                       OBD.GetVehicleSpeed(obd2Data.VehicleSpeed()),
+				Intake_air_temperature:              OBD.GetIntakeAirTemperature(obd2Data.IntakeAirTemperature()),
+				Mass_air_flow:                       OBD.GetMassAirFlow(obd2Data.MassAirFlow()),
+				Fuel_tank_level_input:               OBD.GetFuelTankLevelInput(obd2Data.FuelTankLevelInput()),
+				Absolute_barometric_pressure:        OBD.GetAbsoluteBarometricPressure(obd2Data.AbsoluteBarometricPressure()),
+				Control_module_voltage:              OBD.GetControlModuleVoltage(obd2Data.ControlModuleVoltage()),
+				Throttle_position:                   OBD.GetThrottlePosition(obd2Data.ThrottlePosition()),
+				Ambient_air_temperature:             OBD.GetAmbientAirTemperature(obd2Data.AmbientAirTemperature()),
+				Relative_accelerator_pedal_position: OBD.GetRelativeAcceleratorPedalPosition(obd2Data.RelativeAcceleratorPedalPosition()),
+				Engine_oil_temperature:              OBD.GetEngineOilTemperature(obd2Data.EngineOilTemperature()),
+				Engine_fuel_rate:                    OBD.GetEngineFuelRate(obd2Data.EngineFuelRate()),
+				Service_distance:                    int(obd2Data.ServiceDistance()),
+				Anti_lock_barking_active:            int(obd2Data.AntiLockBarkingActive()),
+				Steering_wheel_angle:                int(obd2Data.SteeringWheelAngle()),
+				Position_of_doors:                   int(obd2Data.PositionOfDoors()),
+				Right_left_turn_signal_light:        int(obd2Data.RightLeftTurnSignalLight()),
+				Alternate_beam_head_light:           int(obd2Data.AlternateBeamHeadLight()),
+				High_beam_head_light:                int(obd2Data.HighBeamHeadLight()),
+			},
+		}
+		data = append(data, endpointData)
 	}
 
 	// response to browser
@@ -146,20 +177,20 @@ func Get_dashboard_realtime_data(c *gin.Context) {
 	})
 }
 
-func Descrypt_mam_response(mam_message string, private_key string) Endpoint_obd2_data {
+func Descrypt_mam_response(mam_message string, private_key string) *OBD.OBD2Meta {
 
-	endpoint_serial := endpoint_deserializer.Endpoint_deserializer(mam_message)
-	ciphertext, _ := hex.DecodeString(endpoint_serial.Ciphertext)
-	plaintext := endpoint_CBCDecrypter.Endpoint_CBCDecrypter(string(ciphertext), private_key, endpoint_serial.IV, endpoint_serial.Timestamp)
+	endpointSerial := endpoint_deserializer.Endpoint_Msg_flatbuffer_deserialize(mam_message)
+	ciphertext := endpointSerial.DataBytes()
+	// hmac := endpointSerial.HmacBytes()
+	iv := endpointSerial.IvBytes()
 
-	var endpoint_data Endpoint_obd2_data
-	err := json.Unmarshal([]byte(plaintext), &endpoint_data)
-	if err != nil {
-		fmt.Println("error:", err)
-	}
+	compressData := endpoint_CBCDecrypter.Endpoint_CBCDecrypter(string(ciphertext), private_key, string(iv))
+
+	obd2MetaFlatbuffer, _ := cbrotli.Decode([]byte(compressData))
 
+	obd2Meta := OBD.GetRootAsOBD2Meta(obd2MetaFlatbuffer, 0)
 	log.Println("Descrypt mam response successfully")
-	return endpoint_data
+	return obd2Meta
 }
 
 func Register_device(c *gin.Context) {

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20201104130540-2e1f801663c6
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-redis/redis v6.15.9+incompatible
+	github.com/google/brotli/go/cbrotli v0.0.0-20201008125033-fcda9db7fd55
+	github.com/google/flatbuffers v1.12.0
 	github.com/onsi/ginkgo v1.14.1 // indirect
 	github.com/onsi/gomega v1.10.2 // indirect
 	github.com/spf13/viper v1.7.1

go.sum

@@ -36,18 +36,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/elastic/go-elasticsearch/v8 v8.0.0-20201104130540-2e1f801663c6 h1:uCGWBmQ1KXirA0E3+6g9i/hqfHOG6Wl+QVrJEUGvfQc=
 github.com/elastic/go-elasticsearch/v8 v8.0.0-20201104130540-2e1f801663c6/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -87,8 +87,12 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/brotli/go/cbrotli v0.0.0-20201008125033-fcda9db7fd55 h1:NUJM1LC//7oSHaLGpvXhdiT+jjESax/2UJUcQCJrTA4=
+github.com/google/brotli/go/cbrotli v0.0.0-20201008125033-fcda9db7fd55/go.mod h1:nOPhAkwVliJdNTkj3gXpljmWhjc4wCaVqbMJcPKWP4s=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w=
+github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -164,12 +168,12 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=

models/endpoint_CBCDecrypter/endpoint_CBCDecrypter.go

@@ -14,7 +14,7 @@ func Unpadding(src []byte) []byte {
 	return src[:n-paddingNum]
 }
 
-func Endpoint_CBCDecrypter(ciphertext_str string, key string, iv_str string, timestamp uint64) string {
+func Endpoint_CBCDecrypter(ciphertext_str string, key string, iv_str string) string {
 
 	var ciphertext = []byte(ciphertext_str)