UPDATED: memory leak with WiFiClientSecure (Telegram) and WiFiClient (MQTT)

Question

The title is quite self explanatory, just like the other question I have posted at Exception 28 thrown on ESP8266-01 when connected to Adafruit MQTT and Telegram

Somebody might say this is the same question, but it's actually a follow-up of it, and I deem this as worthy of a standalone question, since it might help other people as well and it is not really covered anywhere else on the internet. I am also going to post the entirety of my code (just like I did on the previous question), although StackOverflow recommends posting the bare minimum to replicate the error, since I feel like it is needed in its entirety to fully replicate it. (Private data is replaced by the word PRIVATE)

#include "UniversalTelegramBot.h"
#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>
#include <Adafruit_MQTT.h>
#include <Adafruit_MQTT_Client.h>

#define BOTtoken "PRIVATE"
#define fanPin 2
#define myChatId "PRIVATE"
#define AIO_SERVER      "io.adafruit.com"
#define AIO_SERVERPORT  1883
#define AIO_USERNAME  "PRIVATE"
#define AIO_KEY  "PRIVATE"

char ssid[] = "PRIVATE";
char password[] = "PRIVATE";
bool fanState;
unsigned long lastTimeBotRan = 0;
unsigned long checkTime = 1000;
int numNewMessages;
unsigned long timerStartPoint = 0;
bool timerStart;
String chat_id;
String text;
int messagesNumber;
String timerString;
String  Request;
const unsigned long rst = 300000;
boolean MQTT_connect();


WiFiClientSecure telegramClient;
WiFiClient MQTTClient;
UniversalTelegramBot bot(BOTtoken, telegramClient);
Adafruit_MQTT_Client mqtt(&MQTTClient, AIO_SERVER, AIO_SERVERPORT, AIO_USERNAME, AIO_KEY);
Adafruit_MQTT_Subscribe PRIVATE = Adafruit_MQTT_Subscribe(&mqtt, AIO_USERNAME "/feeds/PRIVATE");

void checkUpdates(int numNewMessages) {
  for (int i = 0; i < numNewMessages; i++) {
    chat_id = String(bot.messages[i].chat_id);
    text = bot.messages[i].text;
    String from_name = bot.messages[i].from_name;

    if (chat_id != myChatId) {
      bot.sendMessage(chat_id, "Unauthorized user, please refrain from texting this bot again.", "");
      continue;
    }

    if (text == "/start") {
      bot.sendMessage(chat_id, "Welcome " + from_name + "!\n"
                              "Control your fan remotely!\n\n"
                              "/fanon: switch the fan ON\n"
                              "/fanoff: switch the fan OFF\n"
                              "/state: get the current state of the fan"
                              "/timer15: run fan for 15 minutes"
                              "/timer30: run fan for 30 minutes"
                              "/timer60: run fan for 1 hour"
                              "/timer: run fan for the amount of time you specify", "Markdown");
    }

    if (text == "/fanon") {
      digitalWrite(fanPin, HIGH);
      Serial.println("Fan on");
      fanState = true;
      bot.sendMessage(chat_id, "Your fan is ON", "");
    }

    if (text == "/fanoff") {
      fanState = false;
      timerStart = false;
      digitalWrite(fanPin, LOW);
      Serial.println("Fan off");
      bot.sendMessage(chat_id, "Your fan is OFF", "");
    }

    if (text == "/state") {
      if (digitalRead(2) == HIGH) {
        bot.sendMessage(chat_id, "Your fan is ON", "");
      } else {
        bot.sendMessage(chat_id, "Your fan is OFF", "");
      }
    }

    if (text == "/timer15") {
      timerStartPoint = millis();
      digitalWrite(fanPin, HIGH);
      timerStart = true;
      Serial.print("(/timer15) Fan on at ");
      Serial.println(timerStartPoint);
      bot.sendMessage(chat_id, "Your fan will run for 15 minutes", "");
      launchTimer(15);
    }

    if (text == "/timer30") {
      digitalWrite(fanPin, HIGH);
      timerStart = true;
      timerStartPoint = millis();
      Serial.print("(/timer30) Fan on at ");
      Serial.println(timerStartPoint);
      bot.sendMessage(chat_id, "Your fan will run for 30 minutes", "");
      launchTimer(30);
    }

    if (text == "/timer60") {
      digitalWrite(fanPin, HIGH);
      timerStart = true;
      timerStartPoint = millis();
      Serial.print("(/timer60) Fan on at ");
      Serial.println(timerStartPoint);
      bot.sendMessage(chat_id, "Your fan will run for 1 hour", "");
      launchTimer(60);
    }

    if (text == "/timer") {
      messagesNumber = bot.last_message_received + 1;
      bot.sendMessage(chat_id, "How long do you want the fan to run for? (in minutes)", "");
      Serial.println(messagesNumber);

      while (messagesNumber == (bot.last_message_received + 1)) {
        checkUpdates(bot.getUpdates(bot.last_message_received + 1));
        timerString = bot.messages[i].text;
        yield();
      }

      if (messagesNumber < (bot.last_message_received + 1)) {
        unsigned long timer = timerString.toInt();
        Serial.println(timer);
        digitalWrite(fanPin, HIGH);
        timerStart = true;
        timerStartPoint = millis();
        Serial.print("(/timer) Fan on at ");
        Serial.println(timerStartPoint);
        bot.sendMessage(chat_id, "Your fan will run for " + timerString + " minutes", "");
        launchTimer(timer);
      }
    }
    text = "";
  }
}

void launchTimer(unsigned long timeInMinutes) {
  unsigned long timeInMillis = timeInMinutes * 60 * 1000;

  while (timerStart) {
    checkUpdates(bot.getUpdates(bot.last_message_received + 1));
    if (MQTT_connect()) {
      Adafruit_MQTT_Subscribe *subscription_name;
      while ((subscription_name = mqtt.readSubscription(4000))) {
        if (subscription_name == &PRIVATE) {
          Request = ((char *)PRIVATE.lastread);
          if (Request == "fanon") {
            digitalWrite(fanPin, HIGH);
            Serial.println("(Control panel) Fan on");
            fanState = true;
            bot.sendMessage(myChatId, "Fan turned on through Control Panel", "");
          }
          if (Request == "fanoff") {
            fanState = false;
            timerStart = false;
            Serial.println("(Control panel) Fan off");
            digitalWrite(fanPin, LOW);
            bot.sendMessage(myChatId, "Fan turned off through Control Panel", "");
          }
          if (Request == "timer15") {
            timerStartPoint = millis();
            digitalWrite(fanPin, HIGH);
            timerStart = true;
            Serial.print("(CP /timer15) Fan on at ");
            Serial.println(timerStartPoint);
            bot.sendMessage(myChatId, "Fan turned on for 15 minutes through Control Panel", "");
            launchTimer(15);
          }
          if (Request == "timer30") {
            digitalWrite(fanPin, HIGH);
            timerStart = true;
            timerStartPoint = millis();
            Serial.print("(CP /timer30) Fan on at ");
            Serial.println(timerStartPoint);
            bot.sendMessage(myChatId, "Fan turned on for 30 minutes through Control Panel", "");
            launchTimer(30);
          }
          if (Request == "timer60") {
            digitalWrite(fanPin, HIGH);
            timerStart = true;
            timerStartPoint = millis();
            Serial.print("(CP /timer60) Fan on at ");
            Serial.println(timerStartPoint);
            bot.sendMessage(myChatId, "Fan turned on for 1 hour through Control Panel", "");
            launchTimer(60);
          }
        }
      }
    }
    if (millis() - timerStartPoint > timeInMillis) {
      digitalWrite(fanPin, LOW);
      timerStart = false;
      Serial.print("Timer run out at ");
      Serial.println(millis());
      bot.sendMessage(myChatId, "Fan turned off because timer ran out", "");
    }
    yield();
  }
}

boolean MQTT_connect() {
  int8_t ret;
  if (mqtt.connected()) {
    return true;
  }  uint8_t retries = 3;
  while ((ret = mqtt.connect()) != 0) {
    mqtt.disconnect();
    delay(2000);
    retries--;
    if (retries == 0) {
      return false;
    }
  } return true;
}


void setup() {
  Serial.begin(115200);
  telegramClient.setInsecure();

  WiFi.mode(WIFI_STA);
  WiFi.disconnect();
  delay(100);

  Serial.print("Connecting Wifi: ");
  Serial.println(ssid);
  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    Serial.print(".");
    delay(500);
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.print("IP address: ");
  Serial.println(WiFi.localIP());

  pinMode(fanPin, OUTPUT);
  delay(10);
  digitalWrite(fanPin, LOW);


  Request = "";
  mqtt.subscribe(&PRIVATE);

  if (MQTT_connect()) {
    Serial.println("mqtt connected");
  }
  else {
    Serial.println("mqtt connection failed");
  }

}

void loop() {
  if (millis() - lastTimeBotRan > checkTime) {
    numNewMessages = bot.getUpdates(bot.last_message_received + 1);

    while (numNewMessages) {
      checkUpdates(numNewMessages);
      numNewMessages = bot.getUpdates(bot.last_message_received + 1);
    }
    lastTimeBotRan = millis();
  }

  delay(1000);

  if (MQTT_connect()) {
    Adafruit_MQTT_Subscribe *subscription_name;
    while ((subscription_name = mqtt.readSubscription(4000))) {
      if (subscription_name == &PRIVATE) {
        Request = ((char *)PRIVATE.lastread);
        if (Request == "fanon") {
          digitalWrite(fanPin, HIGH);
          Serial.println("(Control panel) Fan on");
          fanState = true;
          bot.sendMessage(myChatId, "Fan turned on through Control Panel", "");
        }
        if (Request == "fanoff") {
          fanState = false;
          timerStart = false;
          Serial.println("(Control panel) Fan off");
          digitalWrite(fanPin, LOW);
          bot.sendMessage(myChatId, "Fan turned off through Control Panel", "");
        }
        if (Request == "timer15") {
          timerStartPoint = millis();
          digitalWrite(fanPin, HIGH);
          timerStart = true;
          Serial.print("(CP /timer15) Fan on at ");
          Serial.println(timerStartPoint);
          bot.sendMessage(myChatId, "Fan turned on for 15 minutes through Control Panel", "");
          launchTimer(15);
        }
        if (Request == "timer30") {
          digitalWrite(fanPin, HIGH);
          timerStart = true;
          timerStartPoint = millis();
          Serial.print("(CP /timer30) Fan on at ");
          Serial.println(timerStartPoint);
          bot.sendMessage(myChatId, "Fan turned on for 30 minutes through Control Panel", "");
          launchTimer(30);
        }
        if (Request == "timer60") {
          digitalWrite(fanPin, HIGH);
          timerStart = true;
          timerStartPoint = millis();
          Serial.print("(CP /timer60) Fan on at ");
          Serial.println(timerStartPoint);
          bot.sendMessage(myChatId, "Fan turned on for 1 hour through Control Panel", "");
          launchTimer(60);
        }
      }
    }
  }

  if (!mqtt.ping()) {
    mqtt.disconnect();
  }
}

Now, here is the issue: I would like to instantiate both WiFiClient (to connect to Adafruit MQTT) and WiFiClientSecure (to connect to my Telegram bot) without ending up with a crash, followed by an Exception 28 stacktrace (of which I have already posted a "normal" and decoded version in the other question at the link). I know for a fact, after hours of research, that I cannot instantiate two instances of WiFiClientSecure because it would exceed the available heap on an ESP01, and I also know that WiFiClient alone, or WiFiClientSecure alone, never happen to throw said exception.

Nothing that I have found says that WiFiClient and WiFiClientSecure cannot be used together, but maybe I am missing something. Therefore my question is: is it possible to use both WiFiClient and WiFiClientSecure, for two different purposes (respectively, MQTT and Telegram), in the same code?

I look forward to reading any piece of advice, since I am currently at a loss for ideas. Thanks in advance to everybody who will help.

EDIT: for anybody who is interested, here are my latest findings:

Overclocking the CPU of the ESP01 from 80 MHz to 160 MHz helps mitigate the crashes, in fact it only crashed once over the course of 8 hours. Going from a crash every 30 minutes to 8 hours sure is a step forward;
I found out (thanks to romkey for the advice), that it WiFiClient and WiFiClientSecure are NOT incompatible, the problem was caused by the free heap. Freeing up the heap, by removing Strings and turning them to char arrays, got the program not to crash over the course of 13 whole hours. I might test this further and leave it on for even more than 24 hours, to see if it is really solved. In case, I will update.

UPDATE: found the problem and changed the title. The problem lies entirely in heap use of both the libraries. Both the libraries use an enormous amount of heap space and after a while it runs out and causes exception 28, because, as far as I know, the ESP tries to read from the wrong part of its memory, causing it to crash. I will attach the Serial output of the heap throughout just 3 minutes:

As you can see, the first instance has the heap show more than 40k, which is normal, as 51k-ish is the maximum. Then it jumps down to 20000, and later runs out, even though the ESP reclaims its own memory after a couple of minutes.

You're asking the wrong question. "is it possible to use both WiFiClient and WiFiClientSecure?" - of course it is. The question you should ask is "How do I reduce my use of the heap so that I can use both WiFiClient and WiFiClientSecure together?" The two primary things you need to do there are reduce or eliminate your use of `String` objects - they allocate storage from the heap to store strings; instead use C `char*` strings - and use `PROGMEM` for your static strings. Beyond that the Telegram and AdafruitIO libraries may also take up too much space. — romkey, Aug 13 '20 at 17:51
You are absolutely right, I really could have phrased my question better. Talking about the heap, I thought about that and tried to print the free heap over time, but it never went lower than 20000, which, as far as I know, is the maximum on an ESP8266-01 (not sure, feel free to correct me). I will try to remove as many unused Strings as possible and to turn those that are useful into char* strings. Do unused Strings take up space in the heap even if they are never used or called? — GoTVm, Aug 13 '20 at 18:32

Vadim Volodko · Answer 1 · 2021-01-07T11:11:31.827

I also do use both WiFiClient and WiFiClientSecure in my ESP8266 multisensors checking and publishing project. And while my WifiClient is up all the time (for MQTT and ThingSpeak), secured connection (needed for publishing to GoogleSheets more rarely - every 30 minutes) I decided to put into the function. The whole project code is too large (there's 5570 lines), so here's a part of it with comments introducing heap usage:

// heap size before call of the function: 37112
int SendGSCRPT() {
  showHeap();   // 35312
  WiFiClientSecure GoogleClient;
  showHeap();   // 27512
  GoogleClient.setInsecure();
  if (!GoogleClient.connect("script.google.com", 443)) {
    GoogleClient.stop();
    return 0;
  }
  showHeap();  // 9840 !!!
  char sGog[200];
  snprintf(sGog, sizeof(sGog), "GET %s HTTP/1.1\r\nHost: script.google.com\r\nUser-Agent: PRIVATE\r\nConnection: close\r\n\r\n", "PRIVATE");
  GoogleClient.print(sGog);
  Serial.println("request sent!");
  int res = 1;
  GoogleClient.stop();
  showHeap(); // 30104
  return res;
}
// heap size after function releases all the memory taken: 37112 again !!!

UPDATED: memory leak with WiFiClientSecure (Telegram) and WiFiClient (MQTT)

1 Answers1