diff --git a/app/lib/mqtt_messages_handler.rb b/app/lib/mqtt_messages_handler.rb index 86fadfae..62282114 100644 --- a/app/lib/mqtt_messages_handler.rb +++ b/app/lib/mqtt_messages_handler.rb @@ -7,46 +7,46 @@ def handle_topic(topic, message, retry_on_nil_device=true) handshake_device(topic) if topic.to_s.include?('inventory') - handle_inventory(topic, message) - elsif topic.to_s.include?('raw') - handle_readings(topic, parse_raw_readings(message), retry_on_nil_device) - elsif topic.to_s.include?('readings') - handle_readings(topic, message, retry_on_nil_device) - elsif topic.to_s.include?('info') - handle_info(topic, message, retry_on_nil_device) + handle_inventory(message) + return true else - true + device = find_device_for_topic(topic, message, retry_on_nil_device) + return nil if device.nil? + with_device_error_handling(device, topic, message) do + if topic.to_s.include?('raw') + handle_readings(device, parse_raw_readings(message)) + elsif topic.to_s.include?('readings') + handle_readings(device, message) + elsif topic.to_s.include?('info') + handle_info(device, message) + else + true + end + end end end private - def handle_inventory(topic, message) + def handle_inventory(message) DeviceInventory.create({ report: (message rescue nil) }) return true end - def handle_readings(topic, message, retry_on_nil_device) - device = find_device_for_topic(topic, message, retry_on_nil_device) - return nil if device.nil? - + def handle_readings(device, message) parsed = JSON.parse(message) if message data = parsed["data"] if parsed return nil if data.nil? or data&.empty? - data.each do |reading| storer.store(device, reading) end - return true rescue Exception => e Sentry.capture_exception(e) raise e if Rails.env.test? end - def handle_info(topic, message, retry_on_nil_device) - device = find_device_for_topic(topic, message, retry_on_nil_device) - return nil if device.nil? + def handle_info(device, message) json_message = JSON.parse(message) device.update_column(:hardware_info, json_message) return true @@ -88,6 +88,41 @@ def handle_nil_device(topic, message, retry_on_nil_device) end end + def with_device_error_handling(device, topic, message, reraise=true, &block) + begin + block.call + rescue Exception => e + hardware_info = device.hardware_info + Sentry.set_tags({ + "device-id": device.id, + "device-hardware-version": hardware_info&.[]("hw_ver"), + "device-esp-version": hardware_info&.[]("esp_ver"), + "device-sam-version": hardware_info&.[]("sam_ver"), + }) + Sentry.capture_exception(e) + last_error = device.ingest_errors.order(created_at: :desc).first + ingest_error = device.ingest_errors.create({ + topic: topic, + message: message, + error_class: e.class.name, + error_message: e.message, + error_trace: e.full_message + }) + if send_device_error_warnings && (!last_error || last_error.created_at < device_error_warning_threshold) + UserMailer.device_ingest_errors(device.id).deliver_later + end + raise e if reraise + end + end + + def send_device_error_warnings + ENV.fetch("SEND_DEVICE_ERROR_WARNINGS", false) + end + + def device_error_warning_threshold + ENV.fetch("DEVICE_ERROR_WARNING_THRESHOLD_HOURS", "6").to_i.hours.ago + end + def device_token(topic) device_token = topic[/device\/sck\/(.*?)\//m, 1].to_s end diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb index df72aa8a..7c8a71f3 100644 --- a/app/mailers/user_mailer.rb +++ b/app/mailers/user_mailer.rb @@ -32,4 +32,10 @@ def device_stopped_publishing device_id mail to: @user.to_email_s, subject: 'Device stopped publishing', from: "SmartCitizen Notifications - Device " end + def device_ingest_errors(device_id) + @device = Device.find(device_id) + @user = @device.owner + mail to: @user.to_email_s, subject: "Device has errors", from: "SmartCitizen Notifications - Device " + end + end diff --git a/app/models/device.rb b/app/models/device.rb index a8425e10..780c5b65 100644 --- a/app/models/device.rb +++ b/app/models/device.rb @@ -25,6 +25,7 @@ class Device < ActiveRecord::Base has_many :components, dependent: :destroy has_many :sensors, through: :components has_one :postprocessing, dependent: :destroy + has_many :ingest_errors, dependent: :destroy has_and_belongs_to_many :experiments diff --git a/app/models/ingest_error.rb b/app/models/ingest_error.rb new file mode 100644 index 00000000..ddadd02a --- /dev/null +++ b/app/models/ingest_error.rb @@ -0,0 +1,3 @@ +class IngestError < ActiveRecord::Base + belongs_to :device +end diff --git a/app/views/user_mailer/device_ingest_errors.html.erb b/app/views/user_mailer/device_ingest_errors.html.erb new file mode 100644 index 00000000..8bdc9af7 --- /dev/null +++ b/app/views/user_mailer/device_ingest_errors.html.erb @@ -0,0 +1,66 @@ + + + + + + Device has errrors + <%= render "email_css" %> + + + + + + + + + + +
+
+ + + + + + + +
+

+ SCK logo +

+
+ + + + + + + + + + + + + +
+

<%= @user.username %>,

+
+

+ We have encountered errors on processing data from the + device '<%= @device %>'. +

+

+ If these errors persist, you may want to update the device firmware, or get in touch with our support team. +

+
+ Manage your notifications +
+

The Smart Citizen Team

+
+
+ +
+
+ + + diff --git a/db/migrate/20241014052837_create_device_ingest_errors.rb b/db/migrate/20241014052837_create_device_ingest_errors.rb new file mode 100644 index 00000000..6a692e57 --- /dev/null +++ b/db/migrate/20241014052837_create_device_ingest_errors.rb @@ -0,0 +1,13 @@ +class CreateDeviceIngestErrors < ActiveRecord::Migration[6.1] + def change + create_table :ingest_errors do |t| + t.references :device, null: false, foreign_key: true + t.text :topic + t.text :message + t.text :error_class + t.text :error_message + t.text :error_trace + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 7eef4fe3..cc624da1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,8 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_08_12_081108) do +ActiveRecord::Schema.define(version: 2024_10_14_052837) do + # These are extensions that must be enabled in order to support this database enable_extension "adminpack" enable_extension "hstore" @@ -65,7 +66,9 @@ t.string "key" t.integer "bus", default: 1, null: false t.datetime "last_reading_at" - t.index ["device_id", "sensor_id"], name: "index_components_on_device_id_and_sensor_id" + t.index ["device_id", "key"], name: "unique_key_for_device", unique: true + t.index ["device_id", "sensor_id"], name: "index_components_on_device_id_and_sensor_id", unique: true + t.index ["device_id", "sensor_id"], name: "unique_sensor_for_device", unique: true end create_table "devices", id: :serial, force: :cascade do |t| @@ -104,7 +107,7 @@ t.string "hardware_name_override" t.string "hardware_version_override" t.string "hardware_slug_override" - t.boolean "precise_location", default: false, null: false + t.boolean "precise_location", default: true, null: false t.boolean "enable_forwarding", default: false, null: false t.index ["device_token"], name: "index_devices_on_device_token", unique: true t.index ["geohash"], name: "index_devices_on_geohash" @@ -159,6 +162,18 @@ t.index ["sluggable_type"], name: "index_friendly_id_slugs_on_sluggable_type" end + create_table "ingest_errors", force: :cascade do |t| + t.bigint "device_id", null: false + t.text "topic" + t.text "message" + t.text "error_class" + t.text "error_message" + t.text "error_trace" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["device_id"], name: "index_ingest_errors_on_device_id" + end + create_table "measurements", id: :serial, force: :cascade do |t| t.string "name" t.text "description" @@ -336,6 +351,7 @@ add_foreign_key "devices_tags", "devices" add_foreign_key "devices_tags", "tags" add_foreign_key "experiments", "users", column: "owner_id" + add_foreign_key "ingest_errors", "devices" add_foreign_key "postprocessings", "devices" add_foreign_key "sensors", "measurements" add_foreign_key "uploads", "users"