diff --git a/homeassistant/components/mqtt/__init__.py b/homeassistant/components/mqtt/__init__.py
index 498114732df..ad9166e2410 100644
--- a/homeassistant/components/mqtt/__init__.py
+++ b/homeassistant/components/mqtt/__init__.py
@@ -16,8 +16,6 @@ from typing import Any, Callable, List, Optional, Union
 import attr
 import requests.certs
 import voluptuous as vol
-import paho.mqtt.client as mqtt
-from paho.mqtt.matcher import MQTTMatcher
 
 from homeassistant import config_entries
 from homeassistant.components import websocket_api
@@ -725,6 +723,11 @@ class MQTT:
         tls_version: Optional[int],
     ) -> None:
         """Initialize Home Assistant MQTT client."""
+        # We don't import them on the top because some integrations
+        # should be able to optionally rely on MQTT.
+        # pylint: disable=import-outside-toplevel
+        import paho.mqtt.client as mqtt
+
         self.hass = hass
         self.broker = broker
         self.port = port
@@ -786,6 +789,9 @@ class MQTT:
 
         This method is a coroutine.
         """
+        # pylint: disable=import-outside-toplevel
+        import paho.mqtt.client as mqtt
+
         result: int = None
         try:
             result = await self.hass.async_add_job(
@@ -877,6 +883,9 @@ class MQTT:
         Resubscribe to all topics we were subscribed to and publish birth
         message.
         """
+        # pylint: disable=import-outside-toplevel
+        import paho.mqtt.client as mqtt
+
         if result_code != mqtt.CONNACK_ACCEPTED:
             _LOGGER.error(
                 "Unable to connect to the MQTT broker: %s",
@@ -968,6 +977,9 @@ class MQTT:
 
 def _raise_on_error(result_code: int) -> None:
     """Raise error if error result."""
+    # pylint: disable=import-outside-toplevel
+    import paho.mqtt.client as mqtt
+
     if result_code != 0:
         raise HomeAssistantError(
             "Error talking to MQTT: {}".format(mqtt.error_string(result_code))
@@ -976,6 +988,9 @@ def _raise_on_error(result_code: int) -> None:
 
 def _match_topic(subscription: str, topic: str) -> bool:
     """Test if topic matches subscription."""
+    # pylint: disable=import-outside-toplevel
+    from paho.mqtt.matcher import MQTTMatcher
+
     matcher = MQTTMatcher()
     matcher[subscription] = True
     try:
diff --git a/homeassistant/components/mqtt/config_flow.py b/homeassistant/components/mqtt/config_flow.py
index a8a378e723c..d3c6ee819b5 100644
--- a/homeassistant/components/mqtt/config_flow.py
+++ b/homeassistant/components/mqtt/config_flow.py
@@ -3,7 +3,6 @@ from collections import OrderedDict
 import queue
 
 import voluptuous as vol
-import paho.mqtt.client as mqtt
 
 from homeassistant import config_entries
 from homeassistant.const import (
@@ -126,6 +125,8 @@ class FlowHandler(config_entries.ConfigFlow):
 
 def try_connection(broker, port, username, password, protocol="3.1"):
     """Test if we can connect to an MQTT broker."""
+    import paho.mqtt.client as mqtt
+
     if protocol == "3.1":
         proto = mqtt.MQTTv31
     else:
diff --git a/homeassistant/components/mqtt/server.py b/homeassistant/components/mqtt/server.py
index f5d369a75c7..3ed2fb71b14 100644
--- a/homeassistant/components/mqtt/server.py
+++ b/homeassistant/components/mqtt/server.py
@@ -4,8 +4,6 @@ import logging
 import tempfile
 
 import voluptuous as vol
-from hbmqtt.broker import Broker, BrokerException
-from passlib.apps import custom_app_context
 
 from homeassistant.const import EVENT_HOMEASSISTANT_STOP
 import homeassistant.helpers.config_validation as cv
@@ -37,6 +35,9 @@ def async_start(hass, password, server_config):
 
     This method is a coroutine.
     """
+    # pylint: disable=import-outside-toplevel
+    from hbmqtt.broker import Broker, BrokerException
+
     passwd = tempfile.NamedTemporaryFile()
 
     gen_server_config, client_config = generate_config(hass, passwd, password)
@@ -65,6 +66,9 @@ def async_start(hass, password, server_config):
 
 def generate_config(hass, passwd, password):
     """Generate a configuration based on current Home Assistant instance."""
+    # pylint: disable=import-outside-toplevel
+    from passlib.apps import custom_app_context
+
     config = {
         "listeners": {
             "default": {
diff --git a/tests/components/mqtt/test_server.py b/tests/components/mqtt/test_server.py
index 71dff7ef3ac..3627c95040e 100644
--- a/tests/components/mqtt/test_server.py
+++ b/tests/components/mqtt/test_server.py
@@ -19,13 +19,9 @@ class TestMQTT:
         """Stop everything that was started."""
         self.hass.stop()
 
-    @patch(
-        "homeassistant.components.mqtt.server.custom_app_context", Mock(return_value="")
-    )
+    @patch("passlib.apps.custom_app_context", Mock(return_value=""))
     @patch("tempfile.NamedTemporaryFile", Mock(return_value=MagicMock()))
-    @patch(
-        "homeassistant.components.mqtt.server.Broker", Mock(return_value=MagicMock())
-    )
+    @patch("hbmqtt.broker.Broker", Mock(return_value=MagicMock()))
     @patch("hbmqtt.broker.Broker.start", Mock(return_value=mock_coro()))
     @patch("homeassistant.components.mqtt.MQTT")
     def test_creating_config_with_pass_and_no_http_pass(self, mock_mqtt):
@@ -45,13 +41,9 @@ class TestMQTT:
         assert mock_mqtt.mock_calls[1][2]["username"] == "homeassistant"
         assert mock_mqtt.mock_calls[1][2]["password"] == password
 
-    @patch(
-        "homeassistant.components.mqtt.server.custom_app_context", Mock(return_value="")
-    )
+    @patch("passlib.apps.custom_app_context", Mock(return_value=""))
     @patch("tempfile.NamedTemporaryFile", Mock(return_value=MagicMock()))
-    @patch(
-        "homeassistant.components.mqtt.server.Broker", Mock(return_value=MagicMock())
-    )
+    @patch("hbmqtt.broker.Broker", Mock(return_value=MagicMock()))
     @patch("hbmqtt.broker.Broker.start", Mock(return_value=mock_coro()))
     @patch("homeassistant.components.mqtt.MQTT")
     def test_creating_config_with_pass_and_http_pass(self, mock_mqtt):