HIP-134 unique connections (#904)

* point to unique connections proto

* add ingest for unique connections msg

* verify unique conection reports

* start db queries for unique connections

* add UniqueConnectionCounts to CoverageShares

* Update enum name for doc links

* calculate qualified radios oracle boost multiplier

* add test for qualified wifi in bad coverage areas

* add poc and dc rewarder test for qualified wifi hotspots

* spell out received_timestamp in db like other tables

* clear unique connections after rewarding

* make unused struct private

* Move unique connections to its own namespace

while it currently comes from the radio-threshold service, and it feelslike a threshold for a radio to pas for rewarding; current radio thresholds is about service provider boosting, and unique connections is about oracle boosting.

* move unique connections check to mod

* add UniqueConnectionIngestor to mobile-verifier server

* clippy updates

* add banned radio to qualified radio test

* make sure qualified radios are exempt from sp banning

* Add primary key for hotspot_pubkey and received_timestamp

this makes sure we're idempotent when reprocessing reports within the same day.

* Add test to make sure we always grab the latest value from the database for a record

If we get sent a new report in the same day, we always want to use the latest supplied number.

* Add conflict for saving, distinct for getting

If we get sent a new report in the same day, we always want to use the latest supplied number.

* update proto to include gateway metadata v2

* Fix distinct on query

The old distinct on was returning all rows for a pubkey in reverse
order, so the latest value was always inserted into the hashmap last.

Now, we are correctly getting a single value for a pubkey that is the
latest by timestamp.

Thank you Brian.

* fix be inclusive on start and exclusive on end

If we use inclusive on the end date, we may use a previous days record
for the current day when we do not mean to.

* batch insert verified reports

batch inserting is more performant from the IO perspective.

* update proto, add oracle boost hex status to reward_v2

send oracle boosting status in reward_v2. sp boosted status has been
renamed to reduce confusion. orable boost status has been added

* add unique connections check to run rewarder

* add tests for finding reports to continue rewarder

* go back to proto master

coverage_point_calculator/src/hexes.rs

@@ -77,13 +77,18 @@ pub(crate) fn clean_covered_hexes(
 
             // hip-131: if the radio is banned, it automatically gets an assignment_multiplier of 0.0
             // hip-103: if a hex is boosted by a service provider >=1x, the oracle
-            // multiplier will automatically be 1x, regardless of boosted_hex_status.
-            let assignment_multiplier = if oracle_boosting_status == OracleBoostingStatus::Banned {
-                dec!(0)
-            } else if ranked.boosted.is_some() {
-                dec!(1)
-            } else {
-                ranked.assignments.boosting_multiplier()
+            //   multiplier will automatically be 1x, regardless of boosted_hex_status.
+            // hip-134: qualified radios earn full Oracle Boosting rewards
+            let assignment_multiplier = match oracle_boosting_status {
+                OracleBoostingStatus::Qualified if radio_type.is_wifi() => dec!(1),
+                OracleBoostingStatus::Banned => dec!(0),
+                OracleBoostingStatus::Qualified | OracleBoostingStatus::Eligible => {
+                    if ranked.boosted.is_some() {
+                        dec!(1)
+                    } else {
+                        ranked.assignments.boosting_multiplier()
+                    }
+                }
             };
 
             let base_coverage_points =
@@ -209,9 +214,54 @@ mod tests {
         )
         .unwrap();
 
-        dbg!(&covered_hexes);
-
         assert_eq!(dec!(0), covered_hexes[0].assignment_multiplier);
         assert_eq!(dec!(0), covered_hexes[1].assignment_multiplier);
     }
+
+    #[rstest]
+    fn hip134_qualified_radio(
+        #[values(
+            OracleBoostingStatus::Qualified,
+            OracleBoostingStatus::Eligible,
+            OracleBoostingStatus::Banned
+        )]
+        boost_status: OracleBoostingStatus,
+        #[values(
+            RadioType::IndoorCbrs,
+            RadioType::OutdoorCbrs,
+            RadioType::IndoorWifi,
+            RadioType::OutdoorWifi
+        )]
+        radio_type: RadioType,
+    ) {
+        let coverage = RankedCoverage {
+            hotspot_key: vec![1],
+            cbsd_id: None,
+            hex: hextree::Cell::from_raw(0x8c2681a3064edff).unwrap(),
+            rank: 1,
+            signal_level: SignalLevel::High,
+            assignments: HexAssignments {
+                footfall: Assignment::C,
+                landtype: Assignment::C,
+                urbanized: Assignment::C,
+            },
+            boosted: NonZeroU32::new(0),
+        };
+
+        let covered_hexes = clean_covered_hexes(
+            radio_type,
+            SpBoostedHexStatus::Eligible,
+            vec![coverage],
+            boost_status,
+        )
+        .unwrap();
+
+        // Only Qualified WIFI radios should bypass bad assignment multiplier
+        let expected_multiplier = match boost_status {
+            OracleBoostingStatus::Qualified if radio_type.is_wifi() => dec!(1),
+            _ => dec!(0),
+        };
+
+        assert_eq!(expected_multiplier, covered_hexes[0].assignment_multiplier);
+    }
 }

coverage_point_calculator/src/lib.rs

@@ -12,6 +12,8 @@
 //! - [CoveredHex::assignment_multiplier]
 //!   - [HIP-103][oracle-boosting]
 //!     - provider boosted hexes increase oracle boosting to 1x
+//!   - [HIP-134][carrier-offload]
+//!     - serving >25 unique connection increase oracle boosting to 1x
 //!
 //! - [CoveredHex::rank]
 //!   - [HIP-105][hex-limits]
@@ -44,13 +46,14 @@
 //!   - If a Radio is not [BoostedHexStatus::Eligible], boost values are removed before calculations.
 //!   - If a Hex is boosted by a Provider, the Oracle Assignment multiplier is automatically 1x.
 //!
-//! - [ServiceProviderBoostedRewardEligibility]
+//! - [SPBoostedRewardEligibility]
 //!   - Radio must pass at least 1mb of data from 3 unique phones [HIP-84][provider-boosting]
 //!   - Service Provider can invalidate boosted rewards of a hotspot [HIP-125][provider-banning]
 //!
 //! - [OracleBoostingStatus]
 //!   - Eligible: Radio is eligible for normal oracle boosting multipliers
 //!   - Banned: Radio is banned according to hip-131 rules and all assignment_multipliers are 0.0
+//!   - Qualified: Radio serves >25 unique connections, automatic oracle boosting multiplier of 1x
 //!
 //! [modeled-coverage]:        https://github.com/helium/HIP/blob/main/0074-mobile-poc-modeled-coverage-rewards.md#outdoor-radios
 //! [provider-boosting]:       https://github.com/helium/HIP/blob/main/0084-service-provider-hex-boosting.md
@@ -65,6 +68,7 @@
 //! [location-gaming]:         https://github.com/helium/HIP/blob/main/0119-closing-gaming-loopholes-within-the-mobile-network.md
 //! [provider-banning]:        https://github.com/helium/HIP/blob/main/0125-temporary-anti-gaming-measures-for-boosted-hexes.md
 //! [anti-gaming]:             https://github.com/helium/HIP/blob/main/0131-bridging-gap-between-verification-mappers-and-anti-gaming-measures.md
+//! [carrier-offload]:         https://github.com/helium/HIP/blob/main/0134-reward-mobile-carrier-offload-hotspots.md
 //!
 pub use crate::{
     hexes::{CoveredHex, HexPoints},
@@ -136,10 +140,12 @@ pub struct CoveragePoints {
     pub speedtest_multiplier: Decimal,
     /// Input Radio Type
     pub radio_type: RadioType,
-    /// Input ServiceProviderBoostedRewardEligibility
+    /// Input SPBoostedRewardEligibility
     pub service_provider_boosted_reward_eligibility: SPBoostedRewardEligibility,
-    /// Derived Eligibility for Boosted Hex Rewards
-    pub boosted_hex_eligibility: SpBoostedHexStatus,
+    /// Derived Eligibility for Service Provider Boosted Hex Rewards
+    pub sp_boosted_hex_eligibility: SpBoostedHexStatus,
+    /// Derived Eligibility for Oracle Boosted Hex Rewards
+    pub oracle_boosted_hex_eligibility: OracleBoostingStatus,
     /// Speedtests used in calculation
     pub speedtests: Vec<Speedtest>,
     /// Location Trust Scores used in calculation
@@ -157,11 +163,11 @@ impl CoveragePoints {
         speedtests: Vec<Speedtest>,
         location_trust_scores: Vec<LocationTrust>,
         ranked_coverage: Vec<coverage_map::RankedCoverage>,
-        oracle_boosting_status: OracleBoostingStatus,
+        oracle_boost_status: OracleBoostingStatus,
     ) -> Result<CoveragePoints> {
         let location_trust_multiplier = location::multiplier(radio_type, &location_trust_scores);
 
-        let boost_eligibility = SpBoostedHexStatus::new(
+        let sp_boost_eligibility = SpBoostedHexStatus::new(
             radio_type,
             location_trust_multiplier,
             &location_trust_scores,
@@ -170,9 +176,9 @@ impl CoveragePoints {
 
         let covered_hexes = hexes::clean_covered_hexes(
             radio_type,
-            boost_eligibility,
+            sp_boost_eligibility,
             ranked_coverage,
-            oracle_boosting_status,
+            oracle_boost_status,
         )?;
 
         let hex_coverage_points = hexes::calculated_coverage_points(&covered_hexes);
@@ -187,7 +193,8 @@ impl CoveragePoints {
             speedtest_avg,
             radio_type,
             service_provider_boosted_reward_eligibility,
-            boosted_hex_eligibility: boost_eligibility,
+            sp_boosted_hex_eligibility: sp_boost_eligibility,
+            oracle_boosted_hex_eligibility: oracle_boost_status,
             speedtests,
             location_trust_scores,
             covered_hexes,
@@ -230,7 +237,7 @@ impl CoveragePoints {
     }
 
     fn boosted_points(&self) -> Decimal {
-        match self.boosted_hex_eligibility {
+        match self.sp_boosted_hex_eligibility {
             SpBoostedHexStatus::Eligible => self.coverage_points.boosted,
             SpBoostedHexStatus::WifiLocationScoreBelowThreshold(_) => dec!(0),
             SpBoostedHexStatus::AverageAssertedDistanceOverLimit(_) => dec!(0),
@@ -244,6 +251,7 @@ impl CoveragePoints {
 pub enum OracleBoostingStatus {
     Eligible,
     Banned,
+    Qualified,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]

file_store/src/file_info.rs

@@ -173,6 +173,8 @@ pub const VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT: &str =
 pub const PROMOTION_REWARD_INGEST_REPORT: &str = "promotion_reward_ingest_report";
 pub const VERIFIED_PROMOTION_REWARD: &str = "verified_promotion_reward";
 pub const SERVICE_PROVIDER_PROMOTION_FUND: &str = "service_provider_promotion_fund";
+pub const UNIQUE_CONNECTIONS_REPORT: &str = "unique_connections_report";
+pub const VERIFIED_UNIQUE_CONNECTIONS_REPORT: &str = "verified_unique_connections_report";
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Copy, strum::EnumCount)]
 #[serde(rename_all = "snake_case")]
@@ -237,87 +239,13 @@ pub enum FileType {
     RadioUsageStatsIngestReport,
     HexUsageStatsReq,
     RadioUsageStatsReq,
+    UniqueConnectionsReport,
+    VerifiedUniqueConnectionsReport,
 }
 
 impl fmt::Display for FileType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let s = match self {
-            Self::InvalidatedRadioThresholdReq => INVALIDATED_RADIO_THRESHOLD_REQ,
-            Self::InvalidatedRadioThresholdIngestReport => {
-                INVALIDATED_RADIO_THRESHOLD_INGEST_REPORT
-            }
-            Self::VerifiedInvalidatedRadioThresholdIngestReport => {
-                VERIFIED_INVALIDATED_RADIO_THRESHOLD_INGEST_REPORT
-            }
-            Self::RadioThresholdReq => RADIO_THRESHOLD_REQ,
-            Self::RadioThresholdIngestReport => RADIO_THRESHOLD_INGEST_REPORT,
-            Self::VerifiedRadioThresholdIngestReport => VERIFIED_RADIO_THRESHOLD_INGEST_REPORT,
-            Self::SubscriberLocationReq => SUBSCRIBER_LOCATION_REQ,
-            Self::SubscriberLocationIngestReport => SUBSCRIBER_LOCATION_INGEST_REPORT,
-            Self::VerifiedSubscriberLocationIngestReport => {
-                VERIFIED_SUBSCRIBER_LOCATION_INGEST_REPORT
-            }
-            Self::CbrsHeartbeat => CBRS_HEARTBEAT,
-            Self::WifiHeartbeat => WIFI_HEARTBEAT,
-            Self::CellSpeedtest => CELL_SPEEDTEST,
-            Self::VerifiedSpeedtest => VERIFIED_SPEEDTEST,
-            Self::CbrsHeartbeatIngestReport => CELL_HEARTBEAT_INGEST_REPORT,
-            Self::WifiHeartbeatIngestReport => WIFI_HEARTBEAT_INGEST_REPORT,
-            Self::CellSpeedtestIngestReport => CELL_SPEEDTEST_INGEST_REPORT,
-            Self::Entropy => ENTROPY,
-            Self::SubnetworkRewards => SUBNETWORK_REWARDS,
-            Self::EntropyReport => ENTROPY_REPORT,
-            Self::IotBeaconIngestReport => IOT_BEACON_INGEST_REPORT,
-            Self::IotWitnessIngestReport => IOT_WITNESS_INGEST_REPORT,
-            Self::IotPoc => IOT_POC,
-            Self::IotInvalidBeaconReport => IOT_INVALID_BEACON_REPORT,
-            Self::IotInvalidWitnessReport => IOT_INVALID_WITNESS_REPORT,
-            Self::SpeedtestAvg => SPEEDTEST_AVG,
-            Self::ValidatedHeartbeat => VALIDATED_HEARTBEAT,
-            Self::SignedPocReceiptTxn => SIGNED_POC_RECEIPT_TXN,
-            Self::RadioRewardShare => RADIO_REWARD_SHARE,
-            Self::RewardManifest => REWARD_MANIFEST,
-            Self::IotPacketReport => IOT_PACKET_REPORT,
-            Self::IotValidPacket => IOT_VALID_PACKET,
-            Self::InvalidPacket => INVALID_PACKET,
-            Self::NonRewardablePacket => NON_REWARDABLE_PACKET,
-            Self::IotRewardShare => IOT_REWARD_SHARE,
-            Self::DataTransferSessionIngestReport => DATA_TRANSFER_SESSION_INGEST_REPORT,
-            Self::InvalidDataTransferSessionIngestReport => {
-                INVALID_DATA_TRANSFER_SESSION_INGEST_REPORT
-            }
-            Self::ValidDataTransferSession => VALID_DATA_TRANSFER_SESSION,
-            Self::VerifiedDataTransferSession => VERIFIED_DATA_TRANSFER_SESSION,
-            Self::PriceReport => PRICE_REPORT,
-            Self::MobileRewardShare => MOBILE_REWARD_SHARE,
-            Self::MapperMsg => MAPPER_MSG,
-            Self::CoverageObject => COVERAGE_OBJECT,
-            Self::CoverageObjectIngestReport => COVERAGE_OBJECT_INGEST_REPORT,
-            Self::SeniorityUpdate => SENIORITY_UPDATE,
-            Self::BoostedHexUpdate => BOOSTED_HEX_UPDATE,
-            Self::OracleBoostingReport => ORACLE_BOOSTING_REPORT,
-            Self::UrbanizationDataSet => URBANIZATION_DATA_SET,
-            Self::FootfallDataSet => FOOTFALL_DATA_SET,
-            Self::LandtypeDataSet => LANDTYPE_DATA_SET,
-            Self::SPBoostedRewardsBannedRadioIngestReport => SP_BOOSTED_REWARDS_BANNED_RADIO,
-            Self::VerifiedSPBoostedRewardsBannedRadioIngestReport => {
-                VERIFIED_SP_BOOSTED_REWARDS_BANNED_RADIO
-            }
-            Self::SubscriberVerifiedMappingEventIngestReport => {
-                SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT
-            }
-            Self::VerifiedSubscriberVerifiedMappingEventIngestReport => {
-                VERIFIED_SUBSCRIBER_VERIFIED_MAPPING_INGEST_REPORT
-            }
-            Self::PromotionRewardIngestReport => PROMOTION_REWARD_INGEST_REPORT,
-            Self::VerifiedPromotionReward => VERIFIED_PROMOTION_REWARD,
-            Self::ServiceProviderPromotionFund => SERVICE_PROVIDER_PROMOTION_FUND,
-            Self::HexUsageStatsIngestReport => HEX_USAGE_STATS_INGEST_REPORT,
-            Self::RadioUsageStatsIngestReport => RADIO_USAGE_STATS_INGEST_REPORT,
-            Self::HexUsageStatsReq => HEX_USAGE_STATS_REQ,
-            Self::RadioUsageStatsReq => RADIO_USAGE_STATS_REQ,
-        };
-        f.write_str(s)
+        f.write_str(self.to_str())
     }
 }
 
@@ -398,6 +326,8 @@ impl FileType {
             Self::RadioUsageStatsIngestReport => RADIO_USAGE_STATS_INGEST_REPORT,
             Self::HexUsageStatsReq => HEX_USAGE_STATS_REQ,
             Self::RadioUsageStatsReq => RADIO_USAGE_STATS_REQ,
+            Self::UniqueConnectionsReport => UNIQUE_CONNECTIONS_REPORT,
+            Self::VerifiedUniqueConnectionsReport => VERIFIED_UNIQUE_CONNECTIONS_REPORT,
         }
     }
 }
@@ -479,6 +409,8 @@ impl FromStr for FileType {
             RADIO_USAGE_STATS_INGEST_REPORT => Self::RadioUsageStatsIngestReport,
             HEX_USAGE_STATS_REQ => Self::HexUsageStatsReq,
             RADIO_USAGE_STATS_REQ => Self::RadioUsageStatsReq,
+            UNIQUE_CONNECTIONS_REPORT => Self::UniqueConnectionsReport,
+            VERIFIED_UNIQUE_CONNECTIONS_REPORT => Self::VerifiedUniqueConnectionsReport,
             _ => return Err(Error::from(io::Error::from(io::ErrorKind::InvalidInput))),
         };
         Ok(result)

file_store/src/lib.rs

@@ -26,6 +26,7 @@ pub mod speedtest;
 pub mod subscriber_verified_mapping_event;
 pub mod subscriber_verified_mapping_event_ingest_report;
 pub mod traits;
+pub mod unique_connections;
 pub mod usage_counts;
 pub mod verified_subscriber_verified_mapping_event_ingest_report;
 pub mod wifi_heartbeat;

file_store/src/traits/file_sink_write.rs

@@ -263,6 +263,16 @@ impl_file_sink!(
     FileType::RadioUsageStatsIngestReport.to_str(),
     "hotspot_usage_counts_ingest_report"
 );
+impl_file_sink!(
+    poc_mobile::UniqueConnectionsIngestReportV1,
+    FileType::UniqueConnectionsReport.to_str(),
+    "unique_connections_report"
+);
+impl_file_sink!(
+    poc_mobile::VerifiedUniqueConnectionsIngestReportV1,
+    FileType::VerifiedUniqueConnectionsReport.to_str(),
+    "verified_unique_connections_report"
+);
 impl_file_sink!(
     proto::BoostedHexUpdateV1,
     FileType::BoostedHexUpdate.to_str(),