Buffer.h

@@ -0,0 +1,52 @@
+#ifndef BUFFER_H
+#define BUFFER_H
+
+#include <cstring>
+
+class Buffer {
+private:
+    char* data_;
+    size_t size_;
+    size_t currentLength_;
+    const char separator_;
+    boolean overflow=false;
+
+public:
+    Buffer(size_t size, const char separator = '\0')
+            : size_(size), currentLength_(0), separator_(separator) {
+        data_ = new char[size_];
+        data_[0] = '\0';  // Ensure the buffer is null-terminated
+    }
+
+    ~Buffer() {
+        delete[] data_;
+    }
+
+    bool add(const char* str) {
+        size_t strLength = strlen(str);
+        if (currentLength_ + strLength + 1 + (currentLength_ > 0 && separator_ != '\0') < size_) {
+            if (currentLength_ > 0 && separator_ != '\0') {
+                strcat(data_, &separator_);
+                ++currentLength_;
+            }
+            strcat(data_, str);
+            currentLength_ += strLength;
+            return true;
+        } else {
+            overflow=true;
+            return false;
+        }
+    }
+
+    void clear() {
+        data_[0] = '\0';
+        currentLength_ = 0;
+        overflow=false;
+    }
+
+    const char* get() const {
+        return data_;
+    }
+};
+
+#endif

ESPBMS.ino

@@ -1,9 +1,18 @@
 #include "BLEDevice.h"
+#include <ETH.h>
+#include "HQ.h"
 
 static BLEUUID serviceUUID("0000ff00-0000-1000-8000-00805f9b34fb"); //xiaoxiang bms service
 static BLEUUID charUUID_rx("0000ff01-0000-1000-8000-00805f9b34fb"); //xiaoxiang bms rx id
 static BLEUUID charUUID_tx("0000ff02-0000-1000-8000-00805f9b34fb"); //xiaoxiang bms tx id
 
+#define ETH_CLK_MODE    ETH_CLOCK_GPIO0_IN  //  ETH_CLOCK_GPIO17_OUT
+#define ETH_POWER_PIN  16
+#define ETH_TYPE        ETH_PHY_LAN8720
+#define ETH_ADDR        1
+#define ETH_MDC_PIN     23
+#define ETH_MDIO_PIN    18
+
 typedef struct
 {
     byte start;
@@ -12,15 +21,18 @@ typedef struct
     byte dataLen;
 } bmsPacketHeaderStruct;
 
-void setup() {
-    Serial.begin(115200);
-    BLEDevice::init(""); // Initialize BLE device
-}
-
 char currentName[128];
 bool gotBasicInfo;
 bool gotCellInfo;
+static bool eth_ready = false;
 
+void setup() {
+    esp_bt_controller_mem_release(ESP_BT_MODE_CLASSIC_BT);
+    Serial.begin(115200);
+    WiFi.onEvent(WiFiEvent);
+    ETH.begin(ETH_ADDR, ETH_POWER_PIN, ETH_MDC_PIN, ETH_MDIO_PIN, ETH_TYPE, ETH_CLK_MODE);
+    BLEDevice::init("");
+}
 
 void loop() {
     Serial.printf("\r\n\r\n===============================\r\n\r\n");
@@ -33,6 +45,11 @@ void loop() {
 
     Serial.println("Devices found: " + String(foundDevices.getCount()));
 
+    while(!eth_ready){
+        Serial.println("Wait for eth...");
+        delay(250);
+    }
+
     for (int i = 0; i < foundDevices.getCount(); i++) {
         delay(1000);
         Serial.printf("\r\n\r\n===============================\r\n\r\n");
@@ -161,6 +178,7 @@ void loop() {
             }
         }
         pClient->disconnect();
+        hq.poll();
     }
     Serial.println("Reboot!");
     delay(100);
@@ -304,8 +322,8 @@ bool processBasicInfo(byte *data, unsigned int dataLen){
 
     int32_t Watts = Volts * Amps / 1000000; // W
 
-    //Serial.printf("Remaining Capacity: %4.2fAhr\n", ((float)(data[4] * 256 + data[5]))/100);
+    //Serial.printf("Remaining Capacity: %4.2fAh\n", ((float)(data[4] * 256 + data[5]))/100);
-    //Serial.printf("Nominal Capacity: %4.2fAhr\n", ((float)(data[6] * 256 + data[7]))/100);
+    //Serial.printf("Nominal Capacity: %4.2fAh\n", ((float)(data[6] * 256 + data[7]))/100);
 
     uint32_t CapacityRemainAh = ((uint16_t)two_ints_into16(data[4], data[5])) * 10;
     uint8_t CapacityRemainPercent = ((uint8_t)data[19]);
@@ -317,14 +335,14 @@ bool processBasicInfo(byte *data, unsigned int dataLen){
     uint16_t BalanceCodeHigh = (two_ints_into16(data[14], data[15]));
     uint8_t MosfetStatus = ((byte)data[20]);
 
-    Serial.printf(">>>RC.%s.Voltage %f\r\n",currentName, (float)Volts / 1000);
+    hq.send("test.RC.%s.Voltage %f",currentName, (float)Volts / 1000);
-    Serial.printf(">>>RC.%s.Amps %f\r\n",currentName, (float)Amps / 1000);
+    hq.send("test.RC.%s.Amps %f",currentName, (float)Amps / 1000);
-    Serial.printf(">>>RC.%s.Watts %f\r\n",currentName, (float)Watts);
+    hq.send("test.RC.%s.Watts %f",currentName, (float)Watts);
-    Serial.printf(">>>RC.%s.Capacity_Remain_Ah %f\r\n",currentName, (float)CapacityRemainAh / 1000);
+    hq.send("test.RC.%s.Capacity_Remain_Ah %f",currentName, (float)CapacityRemainAh / 1000);
-    Serial.printf(">>>RC.%s.Capacity_Remain_Wh %f\r\n",currentName, ((float)(CapacityRemainAh) / 1000) * ((float)(Volts) / 1000));
+    hq.send("test.RC.%s.Capacity_Remain_Wh %f",currentName, ((float)(CapacityRemainAh) / 1000) * ((float)(Volts) / 1000));
-    Serial.printf(">>>RC.%s.Capacity_Remain_Percent %d\r\n",currentName, CapacityRemainPercent);
+    hq.send("test.RC.%s.Capacity_Remain_Percent %d",currentName, CapacityRemainPercent);
-    Serial.printf(">>>RC.%s.Temp1 %f\r\n",currentName, (float)Temp1 / 10);
+    hq.send("test.RC.%s.Temp1 %f",currentName, (float)Temp1 / 10);
-    Serial.printf(">>>RC.%s.Temp2 %f\r\n",currentName, (float)Temp2 / 10);
+    hq.send("test.RC.%s.Temp2 %f",currentName, (float)Temp2 / 10);
     /*
     Serial.printf("%s Balance Code Low: 0x%x\r\n",currentName, BalanceCodeLow);
     Serial.printf("%s Balance Code High: 0x%x\r\n",currentName, BalanceCodeHigh);
@@ -358,13 +376,13 @@ bool processCellInfo(byte *data, unsigned int dataLen)
             _cellMin = CellVolt;
         }
 
-        Serial.printf(">>>RC.%s.Cell.%d.Voltage %f\r\n",currentName, i+1,(float)CellVolt/1000);
+        hq.send("test.RC.%s.Cell.%d.Voltage %f",currentName, i+1,(float)CellVolt/1000);
     }
 
-    Serial.printf(">>>RC.%s.Max_Cell_Voltage %f\r\n",currentName, (float)_cellMax / 1000);
+    hq.send("test.RC.%s.Max_Cell_Voltage %f",currentName, (float)_cellMax / 1000);
-    Serial.printf(">>>RC.%s.Min_Cell_Voltage %f\r\n",currentName, (float)_cellMin / 1000);
+    hq.send("test.RC.%s.Min_Cell_Voltage %f",currentName, (float)_cellMin / 1000);
-    Serial.printf(">>>RC.%s.Difference_Cell_Voltage %f\r\n",currentName, (float)(_cellMax - _cellMin) / 1000);
+    hq.send("test.RC.%s.Difference_Cell_Voltage %f",currentName, (float)(_cellMax - _cellMin) / 1000);
-    Serial.printf(">>>RC.%s.Average_Cell_Voltage %f\r\n",currentName, (float)(_cellSum / NumOfCells) / 1000);
+    hq.send("test.RC.%s.Average_Cell_Voltage %f",currentName, (float)(_cellSum / NumOfCells) / 1000);
 
     gotCellInfo=true;
 
@@ -412,3 +430,42 @@ int16_t two_ints_into16(int highbyte, int lowbyte) // turns two bytes into a sin
     result = (result | lowbyte); //OR operation, merge the two
     return result;
 }
+
+
+void WiFiEvent(WiFiEvent_t event) {
+    Serial.print("E:");
+    Serial.println(event);
+  switch (event) {
+    case ARDUINO_EVENT_ETH_START:
+      Serial.println("ETH Started");
+      //set eth hostname here
+      ETH.setHostname("esp32-ethernet");
+      break;
+    case ARDUINO_EVENT_ETH_CONNECTED:
+      Serial.println("ETH Connected");
+      break;
+    case ARDUINO_EVENT_ETH_GOT_IP:
+      Serial.print("ETH MAC: ");
+      Serial.print(ETH.macAddress());
+      Serial.print(", IPv4: ");
+      Serial.print(ETH.localIP());
+      if (ETH.fullDuplex()) {
+        Serial.print(", FULL_DUPLEX");
+      }
+      Serial.print(", ");
+      Serial.print(ETH.linkSpeed());
+      Serial.println("Mbps");
+      eth_ready = true;
+      break;
+    case ARDUINO_EVENT_ETH_DISCONNECTED:
+      Serial.println("ETH Disconnected");
+      eth_ready = false;
+      break;
+    case ARDUINO_EVENT_ETH_STOP:
+      Serial.println("ETH Stopped");
+      eth_ready = false;
+      break;
+    default:
+      break;
+  }
+}

HQ.h

@@ -0,0 +1,98 @@
+#ifndef HQ_H
+#define HQ_H
+
+#include <WiFiClientSecure.h>
+#include <HTTPClient.h>
+#include "Buffer.h"
+#include "ca_cert.h"
+
+class HQ {
+public:
+    // Public static function to get the instance of the singleton class
+    static HQ& getInstance() {
+        static HQ instance; // Create a single instance of the class on first call
+        return instance; // Return the single instance on each call
+    }
+
+private:
+    // Private constructor and copy constructor to prevent outside instantiation
+    HQ(){
+        client = new WiFiClientSecure;
+        if(!client) {
+           Serial.printf("Unable to create HTTP client\r\n");
+        }
+        client->setCACert(ca_cert);
+        client->setHandshakeTimeout(5);
+    }
+    HQ(const HQ&) = delete;
+    Buffer txBuffer=Buffer(4096,'&');
+    Buffer rxBuffer=Buffer(1024,'\n');
+    WiFiClientSecure *client;
+
+public:
+    unsigned long httpErrors=0;
+    boolean send(const char *s){
+        Serial.printf("HQ_SEND:%s\r\n",s);
+        return txBuffer.add(s);
+    }
+    template<typename T, typename... Args>
+    boolean send(const char* format, T arg1, Args... args)
+    {
+        static char msg[1024];
+        snprintf(msg, 1024, format, arg1, args...);
+
+        return send(msg);
+    }
+
+    boolean poll(){
+        String payload;
+        if(strlen(txBuffer.get())){
+            payload+=txBuffer.get();
+        }
+        if(payload.isEmpty()){
+            return true;
+        }
+        HTTPClient https;
+        https.setTimeout(1000);
+        https.setConnectTimeout(1000);
+        https.addHeader("Content-Type","application/x-www-form-urlencoded");
+        String url="https://api.ecomotus.co.uk/api/v1/graphite";
+        Serial.printf("Connecting to %s\r\n",url.c_str());
+        if(!https.begin(*client, url)) {  // HTTPS
+            Serial.printf("HTTP Connection Failed\r\n");
+            httpErrors++;
+            return false;
+        }
+        int httpCode=https.POST(payload);
+        if(httpCode==200){
+            txBuffer.clear();
+            // Read all the lines of the reply from server and print them to Serial
+            boolean command=false;
+            char buff[128];
+            int totalSize=0;
+            WiFiClient * stream = https.getStreamPtr();
+            while(stream->available() && totalSize<1000) {
+                size_t size=stream->readBytesUntil('\n',buff,120);
+                totalSize+=size;
+                buff[size]=0;
+                Serial.printf("Response: %s\r\n",buff);
+                if(command==true){
+                    rxBuffer.add(buff);
+                    command=false;
+                }
+                if(0==strcmp(buff,"COMMAND")){
+                    command=true;
+                }
+            }
+            Serial.printf("Got %d bytes\r\n",totalSize);
+        }else{
+            Serial.printf("HTTP Error: %d\r\n",httpCode);
+            httpErrors++;
+            return false;
+        }
+        return true;
+    }
+};
+
+HQ& hq = HQ::getInstance();
+#endif

bin/2graphite

@@ -0,0 +1,62 @@
+import socket
+import sys
+import time
+import serial
+import select
+from collections import deque
+
+def parse_to_graphite(data):
+    timestamp = int(time.time())
+    results = []
+
+    lines = data.strip().split('\n')
+    for line in lines:
+        if line.startswith('>>>'):
+            metric_path = line[3:].strip()  # Remove the ">>>" prefix and any leading/trailing whitespace
+            results.append(f"{metric_path} {timestamp}")
+    return results
+
+def send_to_graphite(data, host='10.6.0.1', port=2003):
+    try:
+        with socket.create_connection((host, port), timeout=1) as sock:
+            while data:
+                message = data.popleft()
+                print(f"Sending data to Graphite: {message}")  # Debug message for sending data
+                sock.sendall(message.encode('utf-8'))
+    except (socket.error, socket.timeout) as e:
+        print(f"Failed to send data, will drop if buffer is full. Error: {e}", file=sys.stderr)
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python script.py <serial_device>")
+        sys.exit(1)
+
+    serial_device = sys.argv[1]
+    ser = serial.Serial(serial_device, 115200, timeout=1)
+    data_buffer = deque(maxlen=100)  # Buffer up to 100 messages
+    print(f"Reading data from {serial_device}, press CTRL+C to quit:")
+
+    try:
+        while True:
+            if select.select([ser], [], [], 0)[0]:  # Check if there is data to read
+                line = ser.readline().decode('utf-8').strip()
+                if line:
+                    print(f"{line}")  # Output raw line to screen
+                    graphite_data = parse_to_graphite(line)
+                    for data in graphite_data:
+                        if len(data_buffer) < data_buffer.maxlen:
+                            data_buffer.append(data + '\n')
+                            print(f"Data queued for Graphite: {data}")  # Debug message for queuing data
+
+            # Try to send data if the buffer is not empty
+            if data_buffer:
+                print("Attempting to send data...")  # Debug message before sending data
+                send_to_graphite(data_buffer)
+
+    except KeyboardInterrupt:
+        print("Exiting...")
+    finally:
+        ser.close()
+
+if __name__ == '__main__':
+    main()

bin/2graphite.py

@@ -1,57 +0,0 @@
-import asyncio
-import aiohttp
-import sys
-import serial_asyncio  # Ensure this package is installed
-
-def parse_to_graphite(data):
-    results = []
-
-    lines = data.strip().split('\n')
-    for line in lines:
-        if line.startswith('>>>'):
-            metric_path = line[3:].strip()  # Remove the ">>>" prefix and any leading/trailing whitespace
-            results.append(metric_path)
-    return results
-
-async def send_to_graphite(data, session, api_url):
-    for message in data:
-        #print(f"Sending data to API: {message}")  # Debug message for sending data
-        try:
-            # Sending raw metric path directly as plain text
-            headers = {'Content-Type': 'text/plain'}
-            async with session.post(api_url, data=message, headers=headers) as response:
-                if response.status != 200:
-                    print(f"Failed to send data: {response.status}", await response.text())
-        except Exception as e:
-            print(f"Error sending data: {e}")
-
-async def handle_serial(reader, api_url):
-    session = aiohttp.ClientSession()
-    try:
-        while True:
-            line = await reader.readline()
-            if not line:
-                break
-            line = line.decode('utf-8')
-            print(line.strip())  # echo output for received line
-            graphite_data = parse_to_graphite(line)
-            if graphite_data:
-                await send_to_graphite(graphite_data, session, api_url)
-    finally:
-        await session.close()
-
-async def main():
-    if len(sys.argv) < 3:
-        print("Usage: python script.py <serial_device> <api_url>")
-        sys.exit(1)
-
-    serial_device = sys.argv[1]
-    api_url = sys.argv[2]
-    baud_rate = 115200  # You can modify this as needed
-
-    # Creating the connection to the serial port
-    reader, _ = await serial_asyncio.open_serial_connection(url=serial_device, baudrate=baud_rate)
-    await handle_serial(reader, api_url)
-
-if __name__ == '__main__':
-    asyncio.run(main())

bin/build

@@ -9,9 +9,18 @@ else
     NAME=`git symbolic-ref --short HEAD`
 fi
 
-echo "#define VERSION \"${NAME}\"" >> compile_flags.h
+if [ ! "$1" ] || [ "$1" == "0" ] ; then
-arduino-cli compile $LIBS -e -b esp32:esp32:esp32 || exit 1
+  echo "#define VERSION \"${NAME}\"" >> compile_flags.h
-mv build/esp32.esp32.esp32/*.ino.bin Compiled/${NAME}.bin
+  arduino-cli compile --build-property build.partitions=huge_app --build-property upload.maximum_size=3145728 -e -b esp32:esp32:esp32 || exit 1
-mv build/esp32.esp32.esp32/*.ino.elf Compiled/${NAME}.elf
+  mv build/esp32.esp32.esp32/*.ino.bin Compiled/${NAME}_HW0.bin
-mv build/esp32.esp32.esp32/*.ino.partitions.bin Compiled/${NAME}.partitions.bin
+  mv build/esp32.esp32.esp32/*.ino.elf Compiled/${NAME}_HW0.elf
-bin/flash Compiled/${NAME}.bin
+  mv build/esp32.esp32.esp32/*.ino.partitions.bin Compiled/${NAME}_HW0.partitions.bin
+  #if [ "$1" ] ; then
+    bin/flash Compiled/${NAME}_HW0.bin
+  #fi
+fi
+
+if [ ! "$1" ] ; then
+    7z a -mx=9 Compiled/${NAME}.elf.7z Compiled/*.elf
+    rm Compiled/*.elf
+fi

ca_cert.h

@@ -0,0 +1,23 @@
+#ifndef CA_CERT_H
+#define CA_CERT_H
+const char* ca_cert = \
+    "-----BEGIN CERTIFICATE-----\n" \
+    "MIIDHTCCAgWgAwIBAgIUWLAb5lCXs4G6QxCaV78EtsRQgkEwDQYJKoZIhvcNAQEL\n" \
+    "BQAwHTEbMBkGA1UEAwwSYXBpLmVjb21vdHVzLmNvLnVrMCAXDTIyMTAxOTE1Mjgx\n" \
+    "N1oYDzIxMjIwOTI1MTUyODE3WjAdMRswGQYDVQQDDBJhcGkuZWNvbW90dXMuY28u\n" \
+    "dWswggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwXfttIwX1y1tSeaiZ\n" \
+    "0LtnQ3q9xosglFsXyoLcctJmOf+zgdHbNNxMH8CRbm4Z3ZQ4ghBoL/1RHaERl5aA\n" \
+    "U7oVxr4MPHn6fWsYrLlaXLIcmL6ZS91woTKLejhf6D991sH2Jt0xVDhqerimnF4p\n" \
+    "Ut1U6rY6Lw7aUAUUldChhzRUAkAcMHApWwzxElAM+KFFleLq63AESkT21xYOO+WG\n" \
+    "2hLTQB+hDcBvN9IQ4Ud1V7AQ/MDKzVvJsn/z+KnslbH246l1w6haJk230UbPizau\n" \
+    "fAWl63O2/xIxPJzWBXJeUuvi+lTqCf+ZVPBU6chpyL4xX+I7vCTBoKmpaI+qAF9F\n" \
+    "2C4HAgMBAAGjUzBRMB0GA1UdDgQWBBTtxYrL2Cg9PVp4wx6MllB/cKXXYjAfBgNV\n" \
+    "HSMEGDAWgBTtxYrL2Cg9PVp4wx6MllB/cKXXYjAPBgNVHRMBAf8EBTADAQH/MA0G\n" \
+    "CSqGSIb3DQEBCwUAA4IBAQBAai8ewCT3Q2CgBMxvDLKQx7YRBNlv1gbUtYq88rvK\n" \
+    "iz6yzGmbPP1Ax5LCv0oRtRdnrz0h2F80tBibS2mJ2tqsLd3277yMN81mHB0qVIrR\n" \
+    "tq9aTzjGHUXgXmcezEgkTLTfISebvCB8jdR7cjvFUaTUKH3MLR3jNAAqU6WLVY6Q\n" \
+    "wCYLKRhTU+aYkDeObOu2fsoph8FwR9gB9D4K0/W78UTiOQxLFJmCqubooNtGLrph\n" \
+    "dz1hmIkYSKH3pdhE3kZwNilYVjyfxq3UFkh2/2J0Fz7vB7eaJE6PptcPJ2KgxTMO\n" \
+    "i7QEQ+jNru8B20F4DrbvEa0IY5wv9mywugBsXg5rcfjs\n" \
+    "-----END CERTIFICATE-----\n";
+#endif

config.h

@@ -0,0 +1 @@
+#define URL "https://api.ecomotus.co.uk/api/v1/graphite"

victron.ino

@@ -185,12 +185,12 @@ void decodeVictron(BLEAdvertisedDevice advertisedDevice) {
       return;
     }
 
-    Serial.printf(">>>RC.MPPT.1.Battery_Volts %f\r\n",batteryVoltage);
+    hq.send("test.RC.MPPT.1.Battery_Volts %f",batteryVoltage);
-    Serial.printf(">>>RC.MPPT.1.Battery_Amps %f\r\n",batteryCurrent);
+    hq.send("test.RC.MPPT.1.Battery_Amps %f",batteryCurrent);
-    Serial.printf(">>>RC.MPPT.1.Battery_Watts %f\r\n",batteryVoltage*batteryCurrent);
+    hq.send("test.RC.MPPT.1.Battery_Watts %f",batteryVoltage*batteryCurrent);
-    Serial.printf(">>>RC.MPPT.1.Solar_Watts %f\r\n",inputPower);
+    hq.send("test.RC.MPPT.1.Solar_Watts %f",inputPower);
-    Serial.printf(">>>RC.MPPT.1.Output_Current %f\r\n",outputCurrent);
+    hq.send("test.RC.MPPT.1.Output_Current %f",outputCurrent);
-    Serial.printf(">>>RC.MPPT.1.Yield %f\r\n",todayYield);
+    hq.send("test.RC.MPPT.1.Yield %f",todayYield);
-    Serial.printf(">>>RC.MPPT.1.State %d\r\n",deviceState);
+    hq.send("test.RC.MPPT.1.State %d",deviceState);
   }
 }