diff --git a/homeassistant/components/smtp/notify.py b/homeassistant/components/smtp/notify.py
index 8a96865ab8d..d592f25a61d 100644
--- a/homeassistant/components/smtp/notify.py
+++ b/homeassistant/components/smtp/notify.py
@@ -136,16 +136,15 @@ class MailNotificationService(BaseNotificationService):
         server = None
         try:
             server = self.connect()
-        except smtplib.socket.gaierror:
+        except (smtplib.socket.gaierror, ConnectionRefusedError):
             _LOGGER.exception(
-                "SMTP server not found (%s:%s). "
-                "Please check the IP address or hostname of your SMTP server",
+                "SMTP server not found or refused connection (%s:%s). "
+                "Please check the IP address, hostname, and availability of your SMTP server.",
                 self._server,
                 self._port,
             )
-            return False
 
-        except (smtplib.SMTPAuthenticationError, ConnectionRefusedError):
+        except smtplib.SMTPAuthenticationError:
             _LOGGER.exception(
                 "Login not possible. "
                 "Please check your setting and/or your credentials"