Merge pull request #328 from fablabbcn/bugfix-duplicate-components-ta…

…ke-2

Bugfix duplicate components take 2

app/lib/mqtt_messages_handler.rb

@@ -54,7 +54,7 @@ def handle_topic(topic, message, retry_on_nil_device=true)
 
   def handle_nil_device(topic, message, retry_on_nil_device)
     orphan_device = OrphanDevice.find_by_device_token(device_token(topic))
-    if !topic.to_s.include?("inventory") && !topic.to_s.include?("bridge") && orphan_device
+    if topic.to_s.include?("info") && !topic.to_s.include?("bridge") && orphan_device
       retry_later(topic, message) if retry_on_nil_device
     end
   end

app/models/storer.rb

@@ -14,7 +14,6 @@ def store device, reading, do_update = true
 
       if do_update
         update_device(device, parsed_reading[:parsed_ts], parsed_reading[:sql_data])
-        ws_publish(device)
       end
 
       forward_reading(device, reading)
@@ -49,14 +48,6 @@ def kairos_publish(reading_data)
     Redis.current.publish('telnet_queue', reading_data.to_json)
   end
 
-  def ws_publish(device)
-    return if Rails.env.test? or device.blank?
-    begin
-      Redis.current.publish("data-received", renderer.render( partial: "v0/devices/device", locals: {device: device, current_user: nil}))
-    rescue
-    end
-  end
-
   private
 
   attr_reader :mqtt_client, :renderer

compose.override.local.yml

@@ -11,8 +11,6 @@ services:
     restart: "no"
   telnet-task:
     restart: "no"
-  push:
-    restart: "no"
   grafana:
     entrypoint: ["echo", "Grafana service disabled in development"]
     restart: "no"

compose.yml

@@ -8,7 +8,6 @@ include:
   - compose/telnet-task.yml
   - compose/mqtt.yml
   - compose/web.yml
-  - compose/push.yml
   - compose/kairos.yml
   - compose/cassandra.yml
-  - compose/grafana.yml
+  - compose/grafana.yml

compose/app.yml

@@ -17,7 +17,6 @@ services:
       - sidekiq
       - mqtt-task
       - telnet-task
-      - push
       #- mqtt
     restart: always
     deploy:
@@ -34,4 +33,4 @@ services:
     logging:
       driver: "json-file"
       options:
-        max-size: "100m"
+        max-size: "100m"

docs/onboarding.md

@@ -174,34 +174,6 @@ This is tiggered when the platform receives the first **"Hello World"** from the
 
 http://codepen.io/pral2a/pen/ObMWjG
 
-### Angular Integration:
-
-The SmartCitizen front-end features already a `push.service.js` that can be extended with minor changes:
-
-https://github.com/fablabbcn/smartcitizen-web/blob/2e08faca25675970d56c0b5cc090670ffff73d47/src/app/core/api/push.service.js
-
-This needs to extend as follows:
-
-```
-function devicesToken(then){
-  socket.on('token-received', then);
-}
-
-function deviceToken(tokenID, scope){
-  devicesToken(function(data){
-    if(tokenID == data.device_token) scope.$emit('token', data);
-  })
-}
-```
-
-And use as follows:
-
-`push.token(vm.kitData.token, $scope);`
-
-By including the following service we will be also able to listen notification everytime the Kit published new data:
-
-`push.device(vm.kitData.id, $scope);`
-
 
 
 

lib/tasks/components.rake

@@ -0,0 +1,40 @@
+namespace :components do
+  task :remove_duplicates => :environment do
+    puts "WARNING: This task is potentially destructive. Before continuing please ensure that you have made a database backup. Press 'y' to continue or any other key to exit."
+    ch = STDIN.getch
+    unless ch == "y"
+      abort
+    end
+
+    pre_migrate_sensor_counts = ActiveRecord::Base.connection.execute(
+      "SELECT device_id, count(distinct sensor_id) FROM components GROUP BY device_id"
+    ).to_a
+
+    devices_and_sensors_with_dupes = ActiveRecord::Base.connection.execute(
+      "SELECT device_id, sensor_id FROM components GROUP BY device_id, sensor_id HAVING count(id) > 1"
+    )
+
+    removed = 0
+
+    devices_and_sensors_with_dupes.each do |record|
+      device_id = record["device_id"]
+      sensor_id = record["sensor_id"]
+      last_updated_component = Component.where(device_id: device_id, sensor_id: sensor_id).order("last_reading_at DESC").first
+      all_components = Component.where(device_id: device_id, sensor_id: sensor_id, key: last_updated_component.key).all
+      components_to_remove = all_components - [last_updated_component]
+      removed += components_to_remove.length
+      components_to_remove.each(&:destroy!)
+    end
+
+      post_migrate_sensor_counts = ActiveRecord::Base.connection.execute(
+        "SELECT device_id, count(distinct sensor_id) FROM components GROUP BY device_id"
+      ).to_a
+
+      if pre_migrate_sensor_counts == post_migrate_sensor_counts
+        puts "components deduplicated ok, #{removed} components deleted for #{devices_and_sensors_with_dupes.to_a.length} device/sensor pairs."
+      else
+        raise "Number of sensors per device pre deduplication does not match number post duplication. Please revert to the db backup and inspect."
+      end
+  end
+end
+

scripts/deploy.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Always pull from master? What if staging should deploy 'dev' branch?
 git pull origin master;
-docker compose pull auth push;
+docker compose pull auth;
 # Accept containers as params. Supports starting only 'app db' f.x.
 docker compose build && docker compose up -d $@
 

scripts/nginx-conf/api.smartcitizen.me.conf

@@ -1,34 +1,9 @@
-upstream docker-push {
-  server push:8000;
-}
-
-server {
-
-  listen 80;
-  listen [::]:80;
-
-  listen 443 ssl;
-  listen [::]:443 ssl;
-  ssl_certificate    /etc/ssl/star_smartcitizen_me.pem;
-  ssl_certificate_key    /etc/ssl/star_smartcitizen_me.key;
-
-  server_name staging-ws.smartcitizen.me ws.smartcitizen.me;
-
-  location / {
-    proxy_pass http://docker-push;
-    proxy_http_version 1.1;
-    proxy_set_header Upgrade $http_upgrade;
-    proxy_set_header Connection "upgrade";
-    proxy_set_header Host $host;
-  }
-}
-
 server {
   server_name kairos.smartcitizen.me staging-kairos.smartcitizen.me;
 
   listen 80;
   listen [::]:80;
-  
+
   listen 443 ssl;
   listen [::]:443 ssl;
   ssl_certificate    /etc/ssl/star_smartcitizen_me.pem;
@@ -47,7 +22,7 @@ server {
 
   listen 80;
   listen [::]:80;
-  
+
   listen 443 ssl;
   listen [::]:443 ssl;
   ssl_certificate    /etc/ssl/star_smartcitizen_me.pem;
@@ -96,7 +71,7 @@ server {
 
   listen 80;
   listen [::]:80;
-  
+
   listen 443 ssl;
   listen [::]:443 ssl;
   ssl_certificate    /etc/ssl/star_smartcitizen_me.pem;

spec/lib/mqtt_messages_handler_spec.rb

@@ -133,8 +133,8 @@
         message_handler.handle_topic(@packet.topic, @hardware_info_packet.payload)
       end
 
-      it 'defers messages with unknown device tokens and an orphan device if retry flag is true' do
-        expect(RetryMQTTMessageJob).to receive(:perform_later).with(@invalid_packet.topic, @invalid_packet.payload)
+      it 'does not defer messages with unknown device tokens and an orphan device if retry flag is true' do
+        expect(RetryMQTTMessageJob).not_to receive(:perform_later).with(@invalid_packet.topic, @invalid_packet.payload)
         OrphanDevice.create(device_token: "invalid_device_token")
         message_handler.handle_topic(@invalid_packet.topic, @invalid_packet.payload)
       end

spec/models/storer_spec.rb

@@ -56,7 +56,6 @@
     it 'stores data to device' do
       # model/storer.rb is not using Kairos, but Redis -> Telnet
       # expect(Kairos).to receive(:http_post_to).with("/datapoints", @karios_data)
-      # expect_any_instance_of(Storer).to receive(:ws_publish)
       expect do
         storer.store(device, @data)
       end.not_to raise_error
@@ -73,8 +72,6 @@
     skip 'updates device without touching updated_at' do
       updated_at = device.updated_at
 
-      expect(storer).to receive(:ws_publish)
-
       storer.store(device, @data)
 
       expect(device.reload.updated_at).to eq(updated_at)