Merge pull request letscontrolit#4887 from TD-er/bugfix/4885_hiddenSSID

[WiFi] Fix connect to Hidden SSID on Mikrotik AP (letscontrolit#4885)

docs/source/Tools/Tools.rst

@@ -155,7 +155,7 @@ All these values are described in great detail in the Advanced section, where th
 * **Use Last Connected AP from RTC**:	``false`` means the ESPEasy node needs to scan at reboot and cannot reuse the last used connection before the reboot.
 * **Extra Wait WiFi Connect**: ``true`` means there is an extra wait upto 1000 msec after initiating a connection to an access point. This can be useful when connecting to some FritzBox access points or routers. (Added: 2023/04/05)
 * **Enable SDK WiFi Auto Reconnect**: ``true`` means the Espressif SDK will automatically attempt a reconnect when a connection is briefly lost. Access points (like TP-Link Omada) with "Band Steering" enabled may trigger a quick disconnect to force nodes to connect on the 5 GHz band. (Added: 2023/04/05)
-
+* **Hidden SSID Slow Connect**: ``true`` Connect per found hidden SSID to an access point. Needed for some APs like Mikrotik. This may slow down connecting to the AP significantly. (Added: 2023/11/20)
 
 
 
@@ -759,6 +759,18 @@ This will act much faster on these disconnect events. However it also seems to s
 Whenever ESPEasy calls for a disconnect, or the disconnect takes longer than such a very brief disconnect initiated by the Band Steering algorithm of the access point, ESPEasy will turn off the WiFi and turn it on again as if "Restart WiFi Lost Conn" was enabled.
 
 
+Hidden SSID Slow Connect
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Added: 2023-11-20
+
+Some access points with hidden SSID do not react to a broadcast connect attempt with a given SSID.
+For example Mikrotik routers and access points only allow connecting to a hidden SSID when specifically addressed.
+This may cause a significant slow down connecting to a hidden AP when there are lots of hidden access points with a relative strong signal.
+
+This is enabled by default.
+
+
 Show JSON
 =========
 

src/src/DataStructs/FactoryDefault_WiFi_NVS.cpp

@@ -16,19 +16,18 @@
 
 
 void FactoryDefault_WiFi_NVS::fromSettings() {
-  bits.IncludeHiddenSSID           = Settings.IncludeHiddenSSID();
-  bits.ApDontForceSetup            = Settings.ApDontForceSetup();
-  bits.DoNotStartAP                = Settings.DoNotStartAP();
-  bits.ForceWiFi_bg_mode           = Settings.ForceWiFi_bg_mode();
-  bits.WiFiRestart_connection_lost = Settings.WiFiRestart_connection_lost();
-  bits.WifiNoneSleep               = Settings.WifiNoneSleep();
-  bits.gratuitousARP               = Settings.gratuitousARP();
-  bits.UseMaxTXpowerForSending     = Settings.UseMaxTXpowerForSending();
-  bits.UseLastWiFiFromRTC          = Settings.UseLastWiFiFromRTC();
-  bits.WaitWiFiConnect             = Settings.WaitWiFiConnect();
-  bits.SDK_WiFi_autoreconnect      = Settings.SDK_WiFi_autoreconnect();
-
-  //    bits.hiddenSSIDslowConnect = Settings.
+  bits.IncludeHiddenSSID              = Settings.IncludeHiddenSSID();
+  bits.ApDontForceSetup               = Settings.ApDontForceSetup();
+  bits.DoNotStartAP                   = Settings.DoNotStartAP();
+  bits.ForceWiFi_bg_mode              = Settings.ForceWiFi_bg_mode();
+  bits.WiFiRestart_connection_lost    = Settings.WiFiRestart_connection_lost();
+  bits.WifiNoneSleep                  = Settings.WifiNoneSleep();
+  bits.gratuitousARP                  = Settings.gratuitousARP();
+  bits.UseMaxTXpowerForSending        = Settings.UseMaxTXpowerForSending();
+  bits.UseLastWiFiFromRTC             = Settings.UseLastWiFiFromRTC();
+  bits.WaitWiFiConnect                = Settings.WaitWiFiConnect();
+  bits.SDK_WiFi_autoreconnect         = Settings.SDK_WiFi_autoreconnect();
+  bits.HiddenSSID_SlowConnectPerBSSID = Settings.HiddenSSID_SlowConnectPerBSSID();
 }
 
 void FactoryDefault_WiFi_NVS::applyToSettings() const {
@@ -43,8 +42,7 @@ void FactoryDefault_WiFi_NVS::applyToSettings() const {
   Settings.UseLastWiFiFromRTC(bits.UseLastWiFiFromRTC);
   Settings.WaitWiFiConnect(bits.WaitWiFiConnect);
   Settings.SDK_WiFi_autoreconnect(bits.SDK_WiFi_autoreconnect);
-
-  // Settings.  (bits.hiddenSSIDslowConnect);
+  Settings.HiddenSSID_SlowConnectPerBSSID(bits.HiddenSSID_SlowConnectPerBSSID);
 }
 
 bool FactoryDefault_WiFi_NVS::applyToSettings_from_NVS(ESPEasy_NVS_Helper& preferences) {

src/src/DataStructs/FactoryDefault_WiFi_NVS.h

@@ -28,18 +28,18 @@ class FactoryDefault_WiFi_NVS {
 
   union {
     struct {
-      uint64_t IncludeHiddenSSID           : 1;
-      uint64_t ApDontForceSetup            : 1;
-      uint64_t DoNotStartAP                : 1;
-      uint64_t ForceWiFi_bg_mode           : 1;
-      uint64_t WiFiRestart_connection_lost : 1;
-      uint64_t WifiNoneSleep               : 1;
-      uint64_t gratuitousARP               : 1;
-      uint64_t UseMaxTXpowerForSending     : 1;
-      uint64_t UseLastWiFiFromRTC          : 1;
-      uint64_t WaitWiFiConnect             : 1;
-      uint64_t SDK_WiFi_autoreconnect      : 1;
-      uint64_t hiddenSSIDslowConnect       : 1;
+      uint64_t IncludeHiddenSSID              : 1;
+      uint64_t ApDontForceSetup               : 1;
+      uint64_t DoNotStartAP                   : 1;
+      uint64_t ForceWiFi_bg_mode              : 1;
+      uint64_t WiFiRestart_connection_lost    : 1;
+      uint64_t WifiNoneSleep                  : 1;
+      uint64_t gratuitousARP                  : 1;
+      uint64_t UseMaxTXpowerForSending        : 1;
+      uint64_t UseLastWiFiFromRTC             : 1;
+      uint64_t WaitWiFiConnect                : 1;
+      uint64_t SDK_WiFi_autoreconnect         : 1;
+      uint64_t HiddenSSID_SlowConnectPerBSSID : 1;
 
       uint64_t unused : 52;
     } bits;

src/src/DataStructs/SettingsStruct.h

@@ -170,17 +170,23 @@ class SettingsStruct_tmpl
 
   // Wait for a second after calling WiFi.begin()
   // Especially useful for some FritzBox routers.
-  bool WaitWiFiConnect() const;
-  void WaitWiFiConnect(bool value);
+  bool WaitWiFiConnect() const { return VariousBits_2.WaitWiFiConnect; }
+  void WaitWiFiConnect(bool value) { VariousBits_2.WaitWiFiConnect = value; }
+
+  // Connect to Hidden SSID using channel and BSSID
+  // This is much slower, but appears to be needed for some access points 
+  // like MikroTik.
+  bool HiddenSSID_SlowConnectPerBSSID() const { return !VariousBits_2.HiddenSSID_SlowConnectPerBSSID; }
+  void HiddenSSID_SlowConnectPerBSSID(bool value) { VariousBits_2.HiddenSSID_SlowConnectPerBSSID = !value; }
 
   // Use Espressif's auto reconnect.
-  bool SDK_WiFi_autoreconnect() const;
-  void SDK_WiFi_autoreconnect(bool value);
+  bool SDK_WiFi_autoreconnect() const { return VariousBits_2.SDK_WiFi_autoreconnect; }
+  void SDK_WiFi_autoreconnect(bool value) { VariousBits_2.SDK_WiFi_autoreconnect = value; }
 
   #if FEATURE_RULES_EASY_COLOR_CODE
   // Inhibit RulesCodeCompletion
-  bool DisableRulesCodeCompletion() const;
-  void DisableRulesCodeCompletion(bool value);
+  bool DisableRulesCodeCompletion() const { return VariousBits_2.DisableRulesCodeCompletion; }
+  void DisableRulesCodeCompletion(bool value) { VariousBits_2.DisableRulesCodeCompletion = value; }
   #endif // if FEATURE_RULES_EASY_COLOR_CODE
 
 
@@ -469,7 +475,46 @@ class SettingsStruct_tmpl
   // Do not rename or move this checksum.
   // Checksum calculation will work "around" this
   uint8_t       md5[16]{}; // Store checksum of the settings.
-  uint32_t      VariousBits2 = 0;
+  union {
+    // VariousBits2 defaults to 0, keep in mind when adding bit lookups.
+    struct {
+      uint32_t WaitWiFiConnect                  : 1; // Bit 00
+      uint32_t SDK_WiFi_autoreconnect           : 1; // Bit 01
+      uint32_t DisableRulesCodeCompletion       : 1; // Bit 02
+      uint32_t HiddenSSID_SlowConnectPerBSSID   : 1; // Bit 03  // inverted
+      uint32_t unused_04                        : 1; // Bit 04
+      uint32_t unused_05                        : 1; // Bit 05
+      uint32_t unused_06                        : 1; // Bit 06
+      uint32_t unused_07                        : 1; // Bit 07
+      uint32_t unused_08                        : 1; // Bit 08
+      uint32_t unused_09                        : 1; // Bit 09
+      uint32_t unused_10                        : 1; // Bit 10
+      uint32_t unused_11                        : 1; // Bit 11
+      uint32_t unused_12                        : 1; // Bit 12
+      uint32_t unused_13                        : 1; // Bit 13
+      uint32_t unused_14                        : 1; // Bit 14
+      uint32_t unused_15                        : 1; // Bit 15
+      uint32_t unused_16                        : 1; // Bit 16
+      uint32_t unused_17                        : 1; // Bit 17
+      uint32_t unused_18                        : 1; // Bit 18
+      uint32_t unused_19                        : 1; // Bit 19
+      uint32_t unused_20                        : 1; // Bit 20
+      uint32_t unused_21                        : 1; // Bit 21
+      uint32_t unused_22                        : 1; // Bit 22
+      uint32_t unused_23                        : 1; // Bit 23
+      uint32_t unused_24                        : 1; // Bit 24
+      uint32_t unused_25                        : 1; // Bit 25
+      uint32_t unused_26                        : 1; // Bit 26
+      uint32_t unused_27                        : 1; // Bit 27
+      uint32_t unused_28                        : 1; // Bit 28
+      uint32_t unused_29                        : 1; // Bit 29
+      uint32_t unused_30                        : 1; // Bit 30
+      uint32_t unused_31                        : 1; // Bit 31
+
+    } VariousBits_2;
+    uint32_t      VariousBits2 = 0;
+  };
+
 
   uint8_t       console_serial_port = DEFAULT_CONSOLE_PORT; 
   int8_t        console_serial_rxpin = DEFAULT_CONSOLE_PORT_RXPIN;

src/src/DataStructs_templ/SettingsStruct.cpp

@@ -334,7 +334,7 @@ void SettingsStruct_tmpl<N_TASKS>::CheckI2Cdevice(bool value) { // Inverted
 }
 #endif // if FEATURE_I2C_DEVICE_CHECK
 */
-
+/*
 template<unsigned int N_TASKS>
 bool SettingsStruct_tmpl<N_TASKS>::WaitWiFiConnect() const { 
   return bitRead(VariousBits2, 0);
@@ -368,7 +368,7 @@ void SettingsStruct_tmpl<N_TASKS>::DisableRulesCodeCompletion(bool value) {
   bitWrite(VariousBits2, 2, value);
 }
 #endif // if FEATURE_RULES_EASY_COLOR_CODE
-
+*/
 
 
 template<unsigned int N_TASKS>

src/src/ESPEasyCore/ESPEasyWiFiEvent.cpp

@@ -139,8 +139,16 @@ void WiFiEvent(WiFiEvent_t event, arduino_event_info_t info) {
         ignoreDisconnectEvent = true;
         #if ESP_IDF_VERSION_MAJOR > 3
         WiFiEventData.markDisconnect(static_cast<WiFiDisconnectReason>(info.wifi_sta_disconnected.reason));
+        if (info.wifi_sta_disconnected.reason == WIFI_REASON_AUTH_EXPIRE) {
+          // See: https://github.com/espressif/arduino-esp32/issues/8877#issuecomment-1807677897
+          WiFiSTAClass::_setStatus(WL_CONNECTION_LOST);
+        }
         #else
         WiFiEventData.markDisconnect(static_cast<WiFiDisconnectReason>(info.disconnected.reason));
+        if (info.disconnected.reason == WIFI_REASON_AUTH_EXPIRE) {
+          // See: https://github.com/espressif/arduino-esp32/issues/8877#issuecomment-1807677897
+          WiFiSTAClass::_setStatus(WL_CONNECTION_LOST);
+        }
         #endif
         WiFi.persistent(false);
         WiFi.disconnect(true);

src/src/ESPEasyCore/ESPEasyWifi.cpp

@@ -297,11 +297,11 @@ bool WiFiConnected() {
 
   if ((WiFiEventData.timerAPstart.isSet()) && WiFiEventData.timerAPstart.timeReached()) {
     if (WiFiEventData.timerAPoff.isSet() && !WiFiEventData.timerAPoff.timeReached()) {
-      // Timer reached, so enable AP mode.
-      if (!WifiIsAP(WiFi.getMode())) {
-        if (!WiFiEventData.wifiConnectAttemptNeeded) {
-          addLog(LOG_LEVEL_INFO, F("WiFi : WiFiConnected(), start AP"));
-          if (!Settings.DoNotStartAP()) {
+      if (!Settings.DoNotStartAP()) {
+        // Timer reached, so enable AP mode.
+        if (!WifiIsAP(WiFi.getMode())) {
+          if (!WiFiEventData.wifiConnectAttemptNeeded) {
+            addLog(LOG_LEVEL_INFO, F("WiFi : WiFiConnected(), start AP"));
             WifiScan(false);
             setAP(true);
           }
@@ -469,12 +469,14 @@ void AttemptWiFiConnect() {
       WiFiEventData.wifiConnectInProgress = true;
       const String key = WiFi_AP_CandidatesList::get_key(candidate.index);
 
-      if (candidate.allowQuickConnect() && !candidate.isHidden) {
+      if ((Settings.HiddenSSID_SlowConnectPerBSSID() || !candidate.isHidden)
+           && candidate.allowQuickConnect()) {
         WiFi.begin(candidate.ssid.c_str(), key.c_str(), candidate.channel, candidate.bssid.mac);
       } else {
         WiFi.begin(candidate.ssid.c_str(), key.c_str());
       }
-      if (Settings.WaitWiFiConnect()) {
+      if (Settings.WaitWiFiConnect() || candidate.isHidden) {
+//        WiFi.waitForConnectResult(candidate.isHidden ? 3000 : 1000);  // https://github.com/arendst/Tasmota/issues/14985
         WiFi.waitForConnectResult(1000);  // https://github.com/arendst/Tasmota/issues/14985
       }
       delay(1);

src/src/Helpers/StringProvider.cpp

@@ -179,6 +179,8 @@ const __FlashStringHelper * getLabel(LabelType::Enum label) {
     case LabelType::PERIODICAL_GRAT_ARP:    return F("Periodical send Gratuitous ARP");
     case LabelType::CONNECTION_FAIL_THRESH: return F("Connection Failure Threshold");
     case LabelType::WAIT_WIFI_CONNECT:      return F("Extra Wait WiFi Connect");
+    case LabelType::CONNECT_HIDDEN_SSID:    return F("Include Hidden SSID");
+    case LabelType::HIDDEN_SSID_SLOW_CONNECT: return F("Hidden SSID Slow Connect");
     case LabelType::SDK_WIFI_AUTORECONNECT: return F("Enable SDK WiFi Auto Reconnect");
 
     case LabelType::BUILD_DESC:             return F("Build");
@@ -463,6 +465,8 @@ String getValue(LabelType::Enum label) {
     case LabelType::PERIODICAL_GRAT_ARP:    return jsonBool(Settings.gratuitousARP());
     case LabelType::CONNECTION_FAIL_THRESH: retval = Settings.ConnectionFailuresThreshold; break;
     case LabelType::WAIT_WIFI_CONNECT:      return jsonBool(Settings.WaitWiFiConnect());
+    case LabelType::CONNECT_HIDDEN_SSID:    return jsonBool(Settings.IncludeHiddenSSID());
+    case LabelType::HIDDEN_SSID_SLOW_CONNECT: return jsonBool(Settings.HiddenSSID_SlowConnectPerBSSID());
     case LabelType::SDK_WIFI_AUTORECONNECT: return jsonBool(Settings.WifiNoneSleep());
 
     case LabelType::BUILD_DESC:             return getSystemBuildString();

src/src/Helpers/StringProvider.h

@@ -130,6 +130,8 @@ struct LabelType {
     PERIODICAL_GRAT_ARP,
     CONNECTION_FAIL_THRESH,
     WAIT_WIFI_CONNECT,
+    HIDDEN_SSID_SLOW_CONNECT,
+    CONNECT_HIDDEN_SSID,
     SDK_WIFI_AUTORECONNECT,
 
     BUILD_DESC,

src/src/PluginStructs/P036_data_struct.cpp

@@ -415,6 +415,12 @@ void P036_data_struct::display_header() {
     display_time(); // only for 128pix wide displays
   }
   display_wifibars();
+
+#ifdef OLEDDISPLAY_DOUBLE_BUFFER
+  // Update only small sections of the display, reducing the amount of data to be sent to the display
+  update_display();
+#endif
+
 }
 
 void P036_data_struct::display_time() {

src/src/WebServer/AdvancedConfigPage.cpp

@@ -131,6 +131,7 @@ void handle_advanced() {
     #endif // if FEATURE_I2C_DEVICE_CHECK
 
     Settings.WaitWiFiConnect(isFormItemChecked(LabelType::WAIT_WIFI_CONNECT));
+    Settings.HiddenSSID_SlowConnectPerBSSID(isFormItemChecked(LabelType::HIDDEN_SSID_SLOW_CONNECT));
     Settings.SDK_WiFi_autoreconnect(isFormItemChecked(LabelType::SDK_WIFI_AUTORECONNECT));
 
 
@@ -386,6 +387,7 @@ void handle_advanced() {
 
   addFormCheckBox(LabelType::WAIT_WIFI_CONNECT,      Settings.WaitWiFiConnect());
   addFormCheckBox(LabelType::SDK_WIFI_AUTORECONNECT, Settings.SDK_WiFi_autoreconnect());
+  addFormCheckBox(LabelType::HIDDEN_SSID_SLOW_CONNECT,      Settings.HiddenSSID_SlowConnectPerBSSID());
 
 
 

src/src/WebServer/ConfigPage.cpp

@@ -80,7 +80,8 @@ void handle_config() {
     copyFormPassword(F("key2"),  SecuritySettings.WifiKey2,  sizeof(SecuritySettings.WifiKey2));
 
     // Hidden SSID
-    Settings.IncludeHiddenSSID(isFormItemChecked(F("hiddenssid")));
+    Settings.IncludeHiddenSSID(isFormItemChecked(LabelType::CONNECT_HIDDEN_SSID));
+    Settings.HiddenSSID_SlowConnectPerBSSID(isFormItemChecked(LabelType::HIDDEN_SSID_SLOW_CONNECT));
 
     // Access point password.
     copyFormPassword(F("apkey"), SecuritySettings.WifiAPKey, sizeof(SecuritySettings.WifiAPKey));
@@ -173,8 +174,11 @@ void handle_config() {
   addFormPasswordBox(F("Fallback WPA Key"), F("key2"), SecuritySettings.WifiKey2, 63);
   addFormNote(F("WPA Key must be at least 8 characters long"));
 
-  addFormCheckBox(F("Include Hidden SSID"), F("hiddenssid"), Settings.IncludeHiddenSSID());
+  addFormCheckBox(LabelType::CONNECT_HIDDEN_SSID,      Settings.IncludeHiddenSSID());
   addFormNote(F("Must be checked to connect to a hidden SSID"));
+
+  addFormCheckBox(LabelType::HIDDEN_SSID_SLOW_CONNECT,      Settings.HiddenSSID_SlowConnectPerBSSID());
+  addFormNote(F("Required for some AP brands like Mikrotik to connect to hidden SSID"));
 
   addFormSeparator(2);
   addFormPasswordBox(F("WPA AP Mode Key"), F("apkey"), SecuritySettings.WifiAPKey, 63);

src/src/WebServer/Markup.cpp

@@ -540,6 +540,21 @@ void addRowLabelValue(LabelType::Enum label) {
   addHtml(getValue(label));
 }
 
+void addRowLabelValues(const LabelType::Enum labels[]) {
+  size_t i = 0;
+  LabelType::Enum cur  = static_cast<const LabelType::Enum>(pgm_read_byte(labels + i));
+
+  while (true) {
+    const LabelType::Enum next = static_cast<const LabelType::Enum>(pgm_read_byte(labels + i + 1));
+    addRowLabelValue(cur);
+    if (next == LabelType::MAX_LABEL) {
+      return;
+    }
+    ++i;
+    cur = next;
+  }
+}
+
 void addRowLabelValue_copy(LabelType::Enum label) {
   addRowLabel_copy(getLabel(label));
   addHtml(getValue(label));

src/src/WebServer/Markup.h

@@ -175,6 +175,8 @@ void addRowLabel(LabelType::Enum label);
 
 void addRowLabelValue(LabelType::Enum label);
 
+void addRowLabelValues(const LabelType::Enum labels[]);
+
 void addRowLabelValue_copy(LabelType::Enum label);
 
 // ********************************************************************************

src/src/WebServer/SetupPage.cpp

@@ -111,7 +111,8 @@ void handle_setup() {
               safe_strncpy(SecuritySettings.WifiKey,  password.c_str(), sizeof(SecuritySettings.WifiKey));
               safe_strncpy(SecuritySettings.WifiSSID, ssid.c_str(),     sizeof(SecuritySettings.WifiSSID));
               // Hidden SSID
-              Settings.IncludeHiddenSSID(isFormItemChecked(F("hiddenssid")));
+              Settings.IncludeHiddenSSID(isFormItemChecked(LabelType::CONNECT_HIDDEN_SSID));
+              Settings.HiddenSSID_SlowConnectPerBSSID(isFormItemChecked(LabelType::HIDDEN_SSID_SLOW_CONNECT));
               addHtmlError(SaveSettings());
               WiFiEventData.wifiSetupConnect         = true;
               WiFiEventData.wifiConnectAttemptNeeded = true;
@@ -200,9 +201,12 @@ void handle_setup() {
 
       addFormHeader(F("Advanced WiFi settings"));
 
-      addFormCheckBox(F("Include Hidden SSID"), F("hiddenssid"), Settings.IncludeHiddenSSID());
+      addFormCheckBox(LabelType::CONNECT_HIDDEN_SSID,      Settings.IncludeHiddenSSID());
       addFormNote(F("Must be checked to connect to a hidden SSID"));
 
+      addFormCheckBox(LabelType::HIDDEN_SSID_SLOW_CONNECT,      Settings.HiddenSSID_SlowConnectPerBSSID());
+      addFormNote(F("Required for some AP brands like Mikrotik to connect to hidden SSID"));
+
       html_BR();
       html_BR();