Make monkey patch work in Python 3.6 (#7848)

* Make monkey patch work in Python 3.6

* Update dockerfiles back to 3.6

* Lint

* Do not set env variable for dockerfile

* Lint

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.5
+FROM python:3.6
 MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
 
 # Uncomment any of the following lines to disable the installation.

homeassistant/__main__.py

@@ -10,6 +10,7 @@ import threading
 
 from typing import Optional, List
 
+from homeassistant import monkey_patch
 from homeassistant.const import (
     __version__,
     EVENT_HOMEASSISTANT_START,
@@ -17,7 +18,6 @@ from homeassistant.const import (
     REQUIRED_PYTHON_VER_WIN,
     RESTART_EXIT_CODE,
 )
-from homeassistant.util.async import run_callback_threadsafe
 
 
 def attempt_use_uvloop():
@@ -310,6 +310,9 @@ def setup_and_run_hass(config_dir: str,
         return None
 
     if args.open_ui:
+        # Imported here to avoid importing asyncio before monkey patch
+        from homeassistant.util.async import run_callback_threadsafe
+
         def open_browser(event):
             """Open the webinterface in a browser."""
             if hass.config.api is not None:
@@ -371,6 +374,13 @@ def main() -> int:
     """Start Home Assistant."""
     validate_python()
 
+    if os.environ.get('HASS_MONKEYPATCH_ASYNCIO') == '1':
+        if sys.version_info[:3] >= (3, 6):
+            monkey_patch.disable_c_asyncio()
+        monkey_patch.patch_weakref_tasks()
+    elif sys.version_info[:3] < (3, 5, 3):
+        monkey_patch.patch_weakref_tasks()
+
     attempt_use_uvloop()
 
     if sys.version_info[:3] < (3, 5, 3):

homeassistant/monkey_patch.py

@@ -0,0 +1,75 @@
+"""Monkey patch Python to work around issues causing segfaults.
+
+Under heavy threading operations that schedule calls into
+the asyncio event loop, Task objects are created. Due to
+a bug in Python, GC may have an issue when switching between
+the threads and objects with __del__ (which various components
+in HASS have).
+
+This monkey-patch removes the weakref.Weakset, and replaces it
+with an object that ignores the only call utilizing it (the
+Task.__init__ which calls _all_tasks.add(self)). It also removes
+the __del__ which could trigger the future objects __del__ at
+unpredictable times.
+
+The side-effect of this manipulation of the Task is that
+Task.all_tasks() is no longer accurate, and there will be no
+warning emitted if a Task is GC'd while in use.
+
+Related Python bugs:
+ - https://bugs.python.org/issue26617
+"""
+import sys
+
+
+def patch_weakref_tasks():
+    """Replace weakref.WeakSet to address Python 3 bug."""
+    # pylint: disable=no-self-use, protected-access, bare-except
+    import asyncio.tasks
+
+    class IgnoreCalls:
+        """Ignore add calls."""
+
+        def add(self, other):
+            """No-op add."""
+            return
+
+    asyncio.tasks.Task._all_tasks = IgnoreCalls()
+    try:
+        del asyncio.tasks.Task.__del__
+    except:
+        pass
+
+
+def disable_c_asyncio():
+    """Disable using C implementation of asyncio.
+
+    Required to be able to apply the weakref monkey patch.
+
+    Requires Python 3.6+.
+    """
+    class AsyncioImportFinder:
+        """Finder that blocks C version of asyncio being loaded."""
+
+        PATH_TRIGGER = '_asyncio'
+
+        def __init__(self, path_entry):
+            if path_entry != self.PATH_TRIGGER:
+                raise ImportError()
+            return
+
+        def find_module(self, fullname, path=None):
+            """Find a module."""
+            if fullname == self.PATH_TRIGGER:
+                # We lint in Py34, exception is introduced in Py36
+                # pylint: disable=undefined-variable
+                raise ModuleNotFoundError()  # noqa
+            return None
+
+    sys.path_hooks.append(AsyncioImportFinder)
+    sys.path.insert(0, AsyncioImportFinder.PATH_TRIGGER)
+
+    try:
+        import _asyncio  # noqa
+    except ImportError:
+        pass

script/bootstrap_server

@@ -7,4 +7,4 @@ set -e
 cd "$(dirname "$0")/.."
 
 echo "Installing test dependencies..."
-python3 -m pip install tox
+python3 -m pip install tox colorlog

virtualization/Docker/Dockerfile.dev

@@ -2,7 +2,7 @@
 # Based on the production Dockerfile, but with development additions.
 # Keep this file as close as possible to the production Dockerfile, so the environments match.
 
-FROM python:3.5
+FROM python:3.6
 MAINTAINER Paulus Schoutsen <Paulus@PaulusSchoutsen.nl>
 
 # Uncomment any of the following lines to disable the installation.