refactor: mqtt data model.

models/mqtt_data.go

@@ -8,6 +8,11 @@ type MqttHeartRateRecord struct {
 	Topic                string  `gorm:"size:255;index" json:"topic"`
 	RegionID             uint32  `gorm:"index" json:"regionId"`
 	GatewayMAC           string  `gorm:"size:32;index" json:"gatewayMac"`
+	GatewaySchemaVersion uint32  `json:"gatewaySchemaVersion"`
+	GatewayActiveUplink  int32   `gorm:"type:int" json:"gatewayActiveUplink"`
+	GatewayCellularIMEI  string  `gorm:"size:64" json:"gatewayCellularImei"`
+	GatewayCellularRSSI  int32   `gorm:"type:int" json:"gatewayCellularRssi"`
+	GatewayCellularBER   int32   `gorm:"type:int" json:"gatewayCellularBer"`
 	BandID               uint32  `gorm:"index" json:"bandId"`
 	BeltAddr             string  `gorm:"size:64;index" json:"beltAddr"`
 	PacketNum            uint32  `gorm:"index" json:"packetNum"`
@@ -16,10 +21,16 @@ type MqttHeartRateRecord struct {
 	IsActive             bool    `json:"isActive"`
 	IsOnSkin             bool    `json:"isOnSkin"`
 	Battery              uint32  `json:"battery"`
+	PacketStatusSource   string  `gorm:"size:16" json:"packetStatusSource"`
 	SignalRSSINeg        float64 `gorm:"type:double precision" json:"signalRssiNeg"`
 	SNR                  float64 `gorm:"type:double precision" json:"snr"`
+	RawSignalRSSIX2Neg   uint32  `json:"rawSignalRssiX2Neg"`
+	RawSnrPktX4          int32   `gorm:"type:int" json:"rawSnrPktX4"`
 	HubBusID             uint32  `json:"hubBusId"`
 	HubSubDevID          uint32  `json:"hubSubDevId"`
+	HubRadioBW           int32   `gorm:"type:int" json:"hubRadioBw"`
+	HubRadioSF           uint32  `json:"hubRadioSf"`
+	HubRadioFrequencyMHz float64 `gorm:"type:double precision" json:"hubRadioFrequencyMHz"`
 	ReceivedAt           int64   `gorm:"type:bigint;index" json:"receivedAt"`
 }
 
@@ -29,14 +40,25 @@ type MqttStepCountRecord struct {
 	Topic                string  `gorm:"size:255;index" json:"topic"`
 	RegionID             uint32  `gorm:"index" json:"regionId"`
 	GatewayMAC           string  `gorm:"size:32;index" json:"gatewayMac"`
+	GatewaySchemaVersion uint32  `json:"gatewaySchemaVersion"`
+	GatewayActiveUplink  int32   `gorm:"type:int" json:"gatewayActiveUplink"`
+	GatewayCellularIMEI  string  `gorm:"size:64" json:"gatewayCellularImei"`
+	GatewayCellularRSSI  int32   `gorm:"type:int" json:"gatewayCellularRssi"`
+	GatewayCellularBER   int32   `gorm:"type:int" json:"gatewayCellularBer"`
 	BandID               uint32  `gorm:"index" json:"bandId"`
 	BeltAddr             string  `gorm:"size:64;index" json:"beltAddr"`
 	PacketNum            uint32  `gorm:"index" json:"packetNum"`
 	StepCount            uint32  `json:"stepCount"`
+	PacketStatusSource   string  `gorm:"size:16" json:"packetStatusSource"`
 	SignalRSSINeg        float64 `gorm:"type:double precision" json:"signalRssiNeg"`
 	SNR                  float64 `gorm:"type:double precision" json:"snr"`
+	RawSignalRSSIX2Neg   uint32  `json:"rawSignalRssiX2Neg"`
+	RawSnrPktX4          int32   `gorm:"type:int" json:"rawSnrPktX4"`
 	HubBusID             uint32  `json:"hubBusId"`
 	HubSubDevID          uint32  `json:"hubSubDevId"`
+	HubRadioBW           int32   `gorm:"type:int" json:"hubRadioBw"`
+	HubRadioSF           uint32  `json:"hubRadioSf"`
+	HubRadioFrequencyMHz float64 `gorm:"type:double precision" json:"hubRadioFrequencyMHz"`
 	ReceivedAt           int64   `gorm:"type:bigint;index" json:"receivedAt"`
 }
 
@@ -46,6 +68,11 @@ type MqttGatewayStatusRecord struct {
 	Topic                     string `gorm:"size:255;index" json:"topic"`
 	RegionID                  uint32 `gorm:"index" json:"regionId"`
 	GatewayMAC                string `gorm:"size:32;index" json:"gatewayMac"`
+	GatewaySchemaVersion      uint32 `json:"gatewaySchemaVersion"`
+	GatewayActiveUplink       int32  `gorm:"type:int" json:"gatewayActiveUplink"`
+	GatewayCellularIMEI       string `gorm:"size:64" json:"gatewayCellularImei"`
+	GatewayCellularRSSI       int32  `gorm:"type:int" json:"gatewayCellularRssi"`
+	GatewayCellularBER        int32  `gorm:"type:int" json:"gatewayCellularBer"`
 	BootCount                 uint32 `json:"bootCount"`
 	UptimeMs                  uint32 `json:"uptimeMs"`
 	DurationMsSinceLastPacket uint32 `json:"durationMsSinceLastPacket"`

mqtt/listener.go

@@ -23,6 +23,14 @@ const (
 	defaultWorkers   = 4
 )
 
+type packetStatusSnapshot struct {
+	source             string
+	signalRSSINeg      float64
+	snr                float64
+	rawSignalRSSIX2Neg uint32
+	rawSnrPktX4        int32
+}
+
 type Listener struct {
 	db      *gorm.DB
 	cfg     config.MQTTConfig
@@ -245,13 +253,15 @@ func (l *Listener) persistTrainingSession(record *models.MqttTrainingSessionReco
 }
 
 func buildHeartRateRecord(measurement *whgw_hrpb.HrMeasurement, topic string, now int64) models.MqttHeartRateRecord {
-	regionID := measurement.GetGatewayInfo().GetRegionId()
+	gatewayInfo := measurement.GetGatewayInfo()
+	hubInfo := measurement.GetHubInfo()
+	regionID := gatewayInfo.GetRegionId()
 	if regionID == 0 {
 		regionID = parseRegionFromTopic(topic)
 	}
-	gatewayMAC := formatMAC(measurement.GetGatewayInfo().GetGatewayMac())
+	gatewayMAC := formatMAC(gatewayInfo.GetGatewayMac())
 	packet := measurement.GetHrPacket()
-	rssi, snr := parsePacketStatus(measurement.GetPacketStatus())
+	status := parsePacketStatus(measurement.GetPacketStatus())
 	beltAddr := fmt.Sprintf("%d-%d", regionID, packet.GetId())
 
 	return models.MqttHeartRateRecord{
@@ -259,6 +269,11 @@ func buildHeartRateRecord(measurement *whgw_hrpb.HrMeasurement, topic string, no
 		Topic:                topic,
 		RegionID:             regionID,
 		GatewayMAC:           gatewayMAC,
+		GatewaySchemaVersion: gatewayInfo.GetExtra().GetSchemaVersion(),
+		GatewayActiveUplink:  int32(gatewayInfo.GetExtra().GetActiveUplink().Number()),
+		GatewayCellularIMEI:  gatewayInfo.GetExtra().GetCellularModem().GetImei(),
+		GatewayCellularRSSI:  gatewayInfo.GetExtra().GetCellularModem().GetRssi(),
+		GatewayCellularBER:   gatewayInfo.GetExtra().GetCellularModem().GetBer(),
 		BandID:               packet.GetId(),
 		BeltAddr:             beltAddr,
 		PacketNum:            packet.GetPacketNum(),
@@ -267,22 +282,30 @@ func buildHeartRateRecord(measurement *whgw_hrpb.HrMeasurement, topic string, no
 		IsActive:             packet.GetStatus().GetIsActive(),
 		IsOnSkin:             packet.GetStatus().GetIsOnSkin(),
 		Battery:              packet.GetStatus().GetBattery(),
-		SignalRSSINeg: rssi,
+		PacketStatusSource:   status.source,
-		SNR:           snr,
+		SignalRSSINeg:        status.signalRSSINeg,
-		HubBusID:      measurement.GetHubInfo().GetBusId(),
+		SNR:                  status.snr,
-		HubSubDevID:   measurement.GetHubInfo().GetSubDevId(),
+		RawSignalRSSIX2Neg:   status.rawSignalRSSIX2Neg,
+		RawSnrPktX4:          status.rawSnrPktX4,
+		HubBusID:             hubInfo.GetBusId(),
+		HubSubDevID:          hubInfo.GetSubDevId(),
+		HubRadioBW:           int32(hubInfo.GetRadioParameters().GetBw().Number()),
+		HubRadioSF:           hubInfo.GetRadioParameters().GetSf(),
+		HubRadioFrequencyMHz: float64(hubInfo.GetRadioParameters().GetFrequencyMhz()),
 		ReceivedAt:           now,
 	}
 }
 
 func buildStepCountRecord(measurement *whgw_hrpb.StepCountMeasurement, topic string, now int64) models.MqttStepCountRecord {
-	regionID := measurement.GetGatewayInfo().GetRegionId()
+	gatewayInfo := measurement.GetGatewayInfo()
+	hubInfo := measurement.GetHubInfo()
+	regionID := gatewayInfo.GetRegionId()
 	if regionID == 0 {
 		regionID = parseRegionFromTopic(topic)
 	}
-	gatewayMAC := formatMAC(measurement.GetGatewayInfo().GetGatewayMac())
+	gatewayMAC := formatMAC(gatewayInfo.GetGatewayMac())
 	packet := measurement.GetStepCountPacket()
-	rssi, snr := parsePacketStatus(measurement.GetPacketStatus())
+	status := parsePacketStatus(measurement.GetPacketStatus())
 	beltAddr := fmt.Sprintf("%d-%d", regionID, packet.GetId())
 
 	return models.MqttStepCountRecord{
@@ -290,30 +313,47 @@ func buildStepCountRecord(measurement *whgw_hrpb.StepCountMeasurement, topic str
 		Topic:                topic,
 		RegionID:             regionID,
 		GatewayMAC:           gatewayMAC,
+		GatewaySchemaVersion: gatewayInfo.GetExtra().GetSchemaVersion(),
+		GatewayActiveUplink:  int32(gatewayInfo.GetExtra().GetActiveUplink().Number()),
+		GatewayCellularIMEI:  gatewayInfo.GetExtra().GetCellularModem().GetImei(),
+		GatewayCellularRSSI:  gatewayInfo.GetExtra().GetCellularModem().GetRssi(),
+		GatewayCellularBER:   gatewayInfo.GetExtra().GetCellularModem().GetBer(),
 		BandID:               packet.GetId(),
 		BeltAddr:             beltAddr,
 		PacketNum:            packet.GetPacketNum(),
 		StepCount:            packet.GetStepCount(),
-		SignalRSSINeg: rssi,
+		PacketStatusSource:   status.source,
-		SNR:           snr,
+		SignalRSSINeg:        status.signalRSSINeg,
-		HubBusID:      measurement.GetHubInfo().GetBusId(),
+		SNR:                  status.snr,
-		HubSubDevID:   measurement.GetHubInfo().GetSubDevId(),
+		RawSignalRSSIX2Neg:   status.rawSignalRSSIX2Neg,
+		RawSnrPktX4:          status.rawSnrPktX4,
+		HubBusID:             hubInfo.GetBusId(),
+		HubSubDevID:          hubInfo.GetSubDevId(),
+		HubRadioBW:           int32(hubInfo.GetRadioParameters().GetBw().Number()),
+		HubRadioSF:           hubInfo.GetRadioParameters().GetSf(),
+		HubRadioFrequencyMHz: float64(hubInfo.GetRadioParameters().GetFrequencyMhz()),
 		ReceivedAt:           now,
 	}
 }
 
 func buildGatewayStatusRecord(status *whgw_hrpb.GatewayStatus, topic string, now int64) models.MqttGatewayStatusRecord {
-	regionID := status.GetInfo().GetRegionId()
+	gatewayInfo := status.GetInfo()
+	regionID := gatewayInfo.GetRegionId()
 	if regionID == 0 {
 		regionID = parseRegionFromTopic(topic)
 	}
-	gatewayMAC := formatMAC(status.GetInfo().GetGatewayMac())
+	gatewayMAC := formatMAC(gatewayInfo.GetGatewayMac())
 
 	return models.MqttGatewayStatusRecord{
 		Identifier:                fmt.Sprintf("gateway:%d:%s:%d:%d:%d", regionID, gatewayMAC, status.GetStat().GetBootCount(), status.GetStat().GetUptimeMs(), status.GetStat().GetRxCount()),
 		Topic:                     topic,
 		RegionID:                  regionID,
 		GatewayMAC:                gatewayMAC,
+		GatewaySchemaVersion:      gatewayInfo.GetExtra().GetSchemaVersion(),
+		GatewayActiveUplink:       int32(gatewayInfo.GetExtra().GetActiveUplink().Number()),
+		GatewayCellularIMEI:       gatewayInfo.GetExtra().GetCellularModem().GetImei(),
+		GatewayCellularRSSI:       gatewayInfo.GetExtra().GetCellularModem().GetRssi(),
+		GatewayCellularBER:        gatewayInfo.GetExtra().GetCellularModem().GetBer(),
 		BootCount:                 status.GetStat().GetBootCount(),
 		UptimeMs:                  status.GetStat().GetUptimeMs(),
 		DurationMsSinceLastPacket: status.GetStat().GetDurationMsSinceLastPacket(),
@@ -325,17 +365,27 @@ func buildGatewayStatusRecord(status *whgw_hrpb.GatewayStatus, topic string, now
 	}
 }
 
-func parsePacketStatus(status *whgw_hrpb.IPacketStatus) (float64, float64) {
+func parsePacketStatus(status *whgw_hrpb.IPacketStatus) packetStatusSnapshot {
 	if status == nil {
-		return 0, 0
+		return packetStatusSnapshot{}
 	}
 	if parsed := status.GetParsed(); parsed != nil {
-		return float64(parsed.GetSignalRssiNeg()), float64(parsed.GetSnrPkt())
+		return packetStatusSnapshot{
+			source:        "parsed",
+			signalRSSINeg: float64(parsed.GetSignalRssiNeg()),
+			snr:           float64(parsed.GetSnrPkt()),
+		}
 	}
 	if raw := status.GetRaw(); raw != nil {
-		return -float64(raw.GetSignalRssiX2Neg()) / 2, float64(raw.GetSnrPktX4()) / 4
+		return packetStatusSnapshot{
+			source:             "raw",
+			signalRSSINeg:      -float64(raw.GetSignalRssiX2Neg()) / 2,
+			snr:                float64(raw.GetSnrPktX4()) / 4,
+			rawSignalRSSIX2Neg: raw.GetSignalRssiX2Neg(),
+			rawSnrPktX4:        raw.GetSnrPktX4(),
 		}
-	return 0, 0
+	}
+	return packetStatusSnapshot{}
 }
 
 func formatMAC(data []byte) string {

proto/hr_packet_v2.proto

@@ -5,22 +5,37 @@ package whgw_hr;
 option go_package = "hr_receiver/proto;whgw_hrpb";
 
 enum HrConfidence {
+  // [0,25)
   ZERO = 0;
+  // [25,50)
   LOW = 1;
+  // [50,75]
   MEDIUM = 2;
+  // (75,100]
   HIGH = 3;
 }
 
 enum LoRaBW {
+  // nobody is using 7.8 kHz bandwidth
+  // so to satisfy protobuf, we use NONE for zero value
   BW_NONE = 0;
+  // 10.4 kHz
   BW_10_4 = 0x08;
+  // 15.6 kHz
   BW_15_6 = 0x01;
+  // 20.8 kHz
   BW_20_8 = 0x09;
+  // 31.25 kHz
   BW_31_25 = 0x02;
+  // 41.7 kHz
   BW_41_7 = 0x0A;
+  // 62.5 kHz
   BW_62_5 = 0x03;
+  // 125.0 kHz
   BW_125_0 = 0x04;
+  // 250.0 kHz
   BW_250_0 = 0x05;
+  // 500.0 kHz
   BW_500_0 = 0x06;
 }
 
@@ -37,6 +52,7 @@ message StatusFlag {
   uint32       battery = 4;
 }
 
+
 message HrPacket {
   StatusFlag status = 1;
   uint32 id = 2;
@@ -51,18 +67,25 @@ message StepCountPacket {
 }
 
 message RawPacketStatus {
+  // Estimation of RSSI of the LoRa signal (after despreading) on last packet received.
+  // to get the actual RSSI in dBm, divide by 2 and negate
   uint32 signal_rssi_x2_neg = 1;
+  // Estimation of RSSI of the LoRa signal (after despreading) on last packet received.
+  // to get the actual SNR in dB, divide by 4
   int32  snr_pkt_x4 = 2;
 }
 
 message PacketStatus {
+  // Estimation of RSSI of the LoRa signal (after despreading) on last packet received.
   float signal_rssi_neg = 1;
+  // Estimation of RSSI of the LoRa signal (after despreading) on last packet received.
   float snr_pkt = 2;
 }
 
 message HubInfo {
   uint32         bus_id = 1;
   uint32         sub_dev_id = 2;
+  /** optional **/
   LoRaParameters radio_parameters = 3;
 }
 
@@ -72,6 +95,23 @@ message RadioData {
   bytes           data = 3;
 }
 
+/**
+
+ NOTIFY (NTF): MISO, unack
+ SIGNAL (SIG): MOSI, unack
+
+ INDICATE (INF): MISO, needs MOSI CONFIRM
+ CONFIRM (CNF): MOSI, confirms MISO INDICATE
+
+ REQUEST (REQ): MOSI, needs MISO RESPONSE
+ RESPONSE (RSP): MISO, responds to MOSI REQUEST
+
+ note that frame type is only about its direction and acknowledge behavior,
+ the actual semantics are dependent on the payload
+
+ *
+ **/
+
 enum PacketKind {
   NTF = 0;
   SIG = 1;
@@ -81,14 +121,41 @@ enum PacketKind {
   RSP = 5;
 }
 
+enum NetworkUplinkKind {
+  NETWORK_UPLINK_UNKNOWN = 0;
+  NETWORK_UPLINK_WIFI = 1;
+  NETWORK_UPLINK_CELLULAR = 2;
+}
+
+message CellularModemInfo {
+  string imei = 1;
+  int32  rssi = 2;
+  int32  ber = 3;
+}
+
+message GatewayInfoExtra {
+  uint32            schema_version = 1;
+  NetworkUplinkKind active_uplink = 2;
+  CellularModemInfo cellular_modem = 3;
+}
+
+message GatewayStateReportExtra {
+  uint32            schema_version = 1;
+  NetworkUplinkKind selected_uplink = 2;
+  NetworkUplinkKind active_uplink = 3;
+  bool              cellular_modem_present = 4;
+}
+
 message GatewayStateReport {
   bool   network_has_connection = 1;
   bool   network_has_ip = 2;
   bool   network_has_mqtt_connection = 3;
   bool   bluetooth_has_connect = 4;
   uint32 error_code = 5;
+  GatewayStateReportExtra extra = 6;
 }
 
+/*** Hub (S) to Gateway (M) ***/
 message HubSlaveOutGatewayMasterInMsg {
   oneof choice {
     RadioData   ntf_radio_data = 1;
@@ -96,12 +163,15 @@ message HubSlaveOutGatewayMasterInMsg {
   }
 }
 
+/*** Gateway (M) to Hub (S) ***/
 message GatewayMasterOutHubSlaveInMsg {
   oneof choice {
     GatewayStateReport sig_gateway_state_report = 1;
   }
 }
 
+
+/*** Gateway to MQTT ***/
 message GatewaySlaveOutCloudMasterInMsg {
   oneof choice {
     HrMeasurement ntf_hr_measurement = 1;
@@ -110,6 +180,8 @@ message GatewaySlaveOutCloudMasterInMsg {
   }
 }
 
+/** note that the endpoint should be v2 **/
+
 message BatteryInfo {
   uint32 voltage_mv = 1;
   uint32 soc_percentage = 2;
@@ -119,6 +191,7 @@ message BatteryInfo {
 message GatewayInfo {
   uint32 region_id = 1;
   bytes  gateway_mac = 2;
+  GatewayInfoExtra extra = 3;
 }
 
 message IPacketStatus {
@@ -143,10 +216,16 @@ message StepCountMeasurement {
 }
 
 message GatewayStatistic {
+  // Number of times the gateway has booted in history
   uint32      boot_count = 1;
+  // Uptime of the device (in ms)
   uint32      uptime_ms = 2;
+  // Duration (in ms) since last packet received
+  // would be the duration since boot if no packets received yet
   uint32      duration_ms_since_last_packet = 3;
+  // Total number of packets received since boot
   uint32      rx_count = 4;
+  // optional, battery info
   BatteryInfo battery_info = 5;
 }
 
@@ -160,10 +239,15 @@ message Status {
   string message = 2;
 }
 
+/** Gateway Configuration Messages **/
+/** Master is Gateway; Slave is config client (bluetooth central, phone etc.) **/
+
 enum GatewayConfigEntryKind {
   REGION_ID = 0;
   WIFI_SSID = 1;
   WIFI_PASSWORD = 2;
+  NETWORK_UPLINK = 3;
+  CELLULAR_APN = 4;
 }
 
 message GatewayConfigEntry {
@@ -171,6 +255,8 @@ message GatewayConfigEntry {
     uint32 region_id = 1;
     string wifi_ssid = 2;
     string wifi_password = 3;
+    NetworkUplinkKind network_uplink = 4;
+    string cellular_apn = 5;
   }
 }
 
@@ -188,3 +274,4 @@ message GatewayConfigMasterInSlaveOutMsg {
     GatewayConfigEntry inf_set_entry = 2;
   }
 }
+