Add tests for OS update and boot slot switching (#4108)

Add test that OS update works - use the whole stack using CLI to update to the
latest stable version (unless executed manually on the latest stable release,
this version should never be the same as the currently tested one).

With this test in place, we can also test command for switching the slots, so
add an extra test for that too.

Fixes #4103

(cherry picked from commit 90d36147f71da7933c0d1d53faba78a1792844e3)

tests/smoke_test/test_os_update.py

@@ -0,0 +1,105 @@
+import json
+import logging
+from time import sleep
+
+import pytest
+from labgrid.driver import ExecutionError
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@pytest.mark.dependency()
+@pytest.mark.timeout(120)
+def test_init(shell, shell_json):
+    def check_container_running(container_name):
+        out = shell.run_check(
+            f"docker container inspect -f '{{{{.State.Status}}}}' {container_name} || true"
+        )
+        return "running" in out
+
+    # wait for important containers first
+    while True:
+        if check_container_running("homeassistant") and check_container_running("hassio_supervisor"):
+            break
+
+        sleep(1)
+
+    # wait for the system ready and Supervisor at the latest version
+    while True:
+        supervisor_info = "\n".join(shell.run_check("ha supervisor info --no-progress --raw-json || true"))
+        # make sure not to fail when Supervisor is restarting
+        supervisor_info = json.loads(supervisor_info) if supervisor_info.startswith("{") else None
+        # make sure not to fail when Supervisor is in setup state
+        supervisor_data = supervisor_info.get("data") if supervisor_info else None
+        if supervisor_data and supervisor_data["version"] == supervisor_data["version_latest"]:
+            output = "\n".join(shell.run_check("ha os info || true"))
+            if "System is not ready" not in output:
+                break
+
+        sleep(5)
+
+
+@pytest.mark.dependency(depends=["test_init"])
+@pytest.mark.timeout(300)
+def test_os_update(shell, shell_json, target):
+    # fetch version info and OTA URL
+    shell.run_check("ha su reload --no-progress")
+
+    # update OS to latest stable - in tests it should never be the same version
+    stable_version = shell_json("curl -sSL https://version.home-assistant.io/stable.json")["hassos"]["ova"]
+
+    # Core (and maybe Supervisor) might be downloaded at this point, so we need to keep trying
+    while True:
+        output = "\n".join(shell.run_check(f"ha os update --no-progress --version {stable_version} || true", timeout=120))
+        if "Don't have an URL for OTA updates" in output:
+            shell.run_check("ha su reload --no-progress")
+        elif "Command completed successfully" in output:
+            break
+
+        sleep(5)
+
+    shell.console.expect("Booting `Slot ")
+
+    # reactivate ShellDriver to handle login again
+    target.deactivate(shell)
+    target.activate(shell)
+
+    # wait for the system to be ready after update
+    while True:
+        output = "\n".join(shell.run_check("ha os info || true"))
+        if "System is not ready" not in output:
+            break
+
+        sleep(1)
+
+    # check the updated version
+    os_info = shell_json("ha os info --no-progress --raw-json")
+    assert os_info["data"]["version"] == stable_version, "OS did not update successfully"
+
+
+@pytest.mark.dependency(depends=["test_os_update"])
+@pytest.mark.timeout(180)
+def test_boot_other_slot(shell, shell_json, target):
+    # switch to the other slot
+    os_info = shell_json("ha os info --no-progress --raw-json")
+    other_version = os_info["data"]["boot_slots"]["A"]["version"]
+
+    shell.run_check(f"ha os boot-slot other --no-progress || true")
+
+    shell.console.expect("Booting `Slot ")
+
+    # reactivate ShellDriver to handle login again
+    target.deactivate(shell)
+    target.activate(shell)
+
+    # wait for the system to be ready after switching slots
+    while True:
+        output = "\n".join(shell.run_check("ha os info || true"))
+        if "System is not ready" not in output:
+            break
+
+        sleep(1)
+
+    # check that the boot slot has changed
+    os_info = shell_json("ha os info --no-progress --raw-json")
+    assert os_info["data"]["version"] == other_version