Poll supervisor job state when creating or restoring a backup (#136891)

* Poll supervisor job state when creating or restoring a backup

* Update tests

* Add tests for create and restore jobs finishing early
This commit is contained in:
Erik Montnemery 2025-01-30 11:09:31 +01:00 committed by GitHub
parent 1b5316b269
commit 52feeedd2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 136 additions and 52 deletions

View File

@ -350,8 +350,9 @@ class SupervisorBackupReaderWriter(BackupReaderWriter):
backup_id = data.get("reference")
backup_complete.set()
unsub = self._async_listen_job_events(backup.job_id, on_job_progress)
try:
unsub = self._async_listen_job_events(backup.job_id, on_job_progress)
await self._get_job_state(backup.job_id, on_job_progress)
await backup_complete.wait()
finally:
unsub()
@ -506,12 +507,13 @@ class SupervisorBackupReaderWriter(BackupReaderWriter):
@callback
def on_job_progress(data: Mapping[str, Any]) -> None:
"""Handle backup progress."""
"""Handle backup restore progress."""
if data.get("done") is True:
restore_complete.set()
unsub = self._async_listen_job_events(job.job_id, on_job_progress)
try:
unsub = self._async_listen_job_events(job.job_id, on_job_progress)
await self._get_job_state(job.job_id, on_job_progress)
await restore_complete.wait()
finally:
unsub()

View File

@ -301,6 +301,28 @@ TEST_BACKUP_DETAILS_5 = supervisor_backups.BackupComplete(
type=TEST_BACKUP_5.type,
)
TEST_JOB_ID = "d17bd02be1f0437fa7264b16d38f700e"
TEST_JOB_NOT_DONE = supervisor_jobs.Job(
name="backup_manager_partial_backup",
reference="1ef41507",
uuid=UUID(TEST_JOB_ID),
progress=0.0,
stage="copy_additional_locations",
done=False,
errors=[],
child_jobs=[],
)
TEST_JOB_DONE = supervisor_jobs.Job(
name="backup_manager_partial_backup",
reference="1ef41507",
uuid=UUID(TEST_JOB_ID),
progress=0.0,
stage="copy_additional_locations",
done=True,
errors=[],
child_jobs=[],
)
@pytest.fixture(autouse=True)
def fixture_supervisor_environ() -> Generator[None]:
@ -813,8 +835,9 @@ async def test_reader_writer_create(
) -> None:
"""Test generating a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
await client.send_json_auto_id({"type": "backup/subscribe_events"})
response = await client.receive_json()
@ -836,7 +859,7 @@ async def test_reader_writer_create(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
supervisor_client.backups.partial_backup.assert_called_once_with(
expected_supervisor_options
@ -847,7 +870,7 @@ async def test_reader_writer_create(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123", "reference": "test_slug"},
"data": {"done": True, "uuid": TEST_JOB_ID, "reference": "test_slug"},
},
}
)
@ -877,6 +900,66 @@ async def test_reader_writer_create(
assert response["event"] == {"manager_state": "idle"}
@pytest.mark.usefixtures("hassio_client", "setup_integration")
async def test_reader_writer_create_job_done(
hass: HomeAssistant,
hass_ws_client: WebSocketGenerator,
supervisor_client: AsyncMock,
) -> None:
"""Test generating a backup, and backup job finishes early."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS
supervisor_client.jobs.get_job.return_value = TEST_JOB_DONE
await client.send_json_auto_id({"type": "backup/subscribe_events"})
response = await client.receive_json()
assert response["event"] == {"manager_state": "idle"}
response = await client.receive_json()
assert response["success"]
await client.send_json_auto_id(
{"type": "backup/generate", "agent_ids": ["hassio.local"], "name": "Test"}
)
response = await client.receive_json()
assert response["event"] == {
"manager_state": "create_backup",
"reason": None,
"stage": None,
"state": "in_progress",
}
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
supervisor_client.backups.partial_backup.assert_called_once_with(
DEFAULT_BACKUP_OPTIONS
)
response = await client.receive_json()
assert response["event"] == {
"manager_state": "create_backup",
"reason": None,
"stage": "upload_to_agents",
"state": "in_progress",
}
response = await client.receive_json()
assert response["event"] == {
"manager_state": "create_backup",
"reason": None,
"stage": None,
"state": "completed",
}
supervisor_client.backups.download_backup.assert_not_called()
supervisor_client.backups.remove_backup.assert_not_called()
response = await client.receive_json()
assert response["event"] == {"manager_state": "idle"}
@pytest.mark.usefixtures("hassio_client")
@pytest.mark.parametrize(
(
@ -1006,7 +1089,7 @@ async def test_reader_writer_create_per_agent_encryption(
for i in range(1, 4)
],
)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = replace(
TEST_BACKUP_DETAILS,
locations=create_locations,
@ -1018,6 +1101,7 @@ async def test_reader_writer_create_per_agent_encryption(
for location in create_locations
},
)
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
supervisor_client.mounts.info.return_value = mounts
assert await async_setup_component(hass, BACKUP_DOMAIN, {BACKUP_DOMAIN: {}})
@ -1050,7 +1134,7 @@ async def test_reader_writer_create_per_agent_encryption(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
supervisor_client.backups.partial_backup.assert_called_once_with(
replace(
@ -1065,7 +1149,7 @@ async def test_reader_writer_create_per_agent_encryption(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123", "reference": "test_slug"},
"data": {"done": True, "uuid": TEST_JOB_ID, "reference": "test_slug"},
},
}
)
@ -1176,7 +1260,8 @@ async def test_reader_writer_create_missing_reference_error(
) -> None:
"""Test missing reference error when generating a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
await client.send_json_auto_id({"type": "backup/subscribe_events"})
response = await client.receive_json()
@ -1197,7 +1282,7 @@ async def test_reader_writer_create_missing_reference_error(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
assert supervisor_client.backups.partial_backup.call_count == 1
@ -1206,7 +1291,7 @@ async def test_reader_writer_create_missing_reference_error(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123"},
"data": {"done": True, "uuid": TEST_JOB_ID},
},
}
)
@ -1248,8 +1333,9 @@ async def test_reader_writer_create_download_remove_error(
) -> None:
"""Test download and remove error when generating a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS_5
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
method_mock = getattr(supervisor_client.backups, method)
method_mock.side_effect = exception
@ -1282,7 +1368,7 @@ async def test_reader_writer_create_download_remove_error(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
assert supervisor_client.backups.partial_backup.call_count == 1
@ -1291,7 +1377,7 @@ async def test_reader_writer_create_download_remove_error(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123", "reference": "test_slug"},
"data": {"done": True, "uuid": TEST_JOB_ID, "reference": "test_slug"},
},
}
)
@ -1334,8 +1420,9 @@ async def test_reader_writer_create_info_error(
) -> None:
"""Test backup info error when generating a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.side_effect = exception
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
remote_agent = BackupAgentTest("remote")
await _setup_backup_platform(
@ -1366,7 +1453,7 @@ async def test_reader_writer_create_info_error(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
assert supervisor_client.backups.partial_backup.call_count == 1
@ -1375,7 +1462,7 @@ async def test_reader_writer_create_info_error(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123", "reference": "test_slug"},
"data": {"done": True, "uuid": TEST_JOB_ID, "reference": "test_slug"},
},
}
)
@ -1408,8 +1495,9 @@ async def test_reader_writer_create_remote_backup(
) -> None:
"""Test generating a backup which will be uploaded to a remote agent."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS_5
supervisor_client.jobs.get_job.return_value = TEST_JOB_NOT_DONE
remote_agent = BackupAgentTest("remote")
await _setup_backup_platform(
@ -1440,7 +1528,7 @@ async def test_reader_writer_create_remote_backup(
response = await client.receive_json()
assert response["success"]
assert response["result"] == {"backup_job_id": "abc123"}
assert response["result"] == {"backup_job_id": TEST_JOB_ID}
supervisor_client.backups.partial_backup.assert_called_once_with(
replace(DEFAULT_BACKUP_OPTIONS, location=[LOCATION_CLOUD_BACKUP]),
@ -1451,7 +1539,7 @@ async def test_reader_writer_create_remote_backup(
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123", "reference": "test_slug"},
"data": {"done": True, "uuid": TEST_JOB_ID, "reference": "test_slug"},
},
}
)
@ -1510,7 +1598,7 @@ async def test_reader_writer_create_wrong_parameters(
) -> None:
"""Test generating a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_backup.return_value.job_id = "abc123"
supervisor_client.backups.partial_backup.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS
await client.send_json_auto_id({"type": "backup/subscribe_events"})
@ -1614,17 +1702,33 @@ async def test_agent_receive_remote_backup(
)
@pytest.mark.parametrize(
("get_job_result", "supervisor_events"),
[
(
TEST_JOB_NOT_DONE,
[{"event": "job", "data": {"done": True, "uuid": TEST_JOB_ID}}],
),
(
TEST_JOB_DONE,
[],
),
],
)
@pytest.mark.usefixtures("hassio_client", "setup_integration")
async def test_reader_writer_restore(
hass: HomeAssistant,
hass_ws_client: WebSocketGenerator,
supervisor_client: AsyncMock,
get_job_result: supervisor_jobs.Job,
supervisor_events: list[dict[str, Any]],
) -> None:
"""Test restoring a backup."""
client = await hass_ws_client(hass)
supervisor_client.backups.partial_restore.return_value.job_id = "abc123"
supervisor_client.backups.partial_restore.return_value.job_id = TEST_JOB_ID
supervisor_client.backups.list.return_value = [TEST_BACKUP]
supervisor_client.backups.backup_info.return_value = TEST_BACKUP_DETAILS
supervisor_client.jobs.get_job.return_value = get_job_result
await client.send_json_auto_id({"type": "backup/subscribe_events"})
response = await client.receive_json()
@ -1657,17 +1761,10 @@ async def test_reader_writer_restore(
),
)
await client.send_json_auto_id(
{
"type": "supervisor/event",
"data": {
"event": "job",
"data": {"done": True, "uuid": "abc123"},
},
}
)
response = await client.receive_json()
assert response["success"]
for event in supervisor_events:
await client.send_json_auto_id({"type": "supervisor/event", "data": event})
response = await client.receive_json()
assert response["success"]
response = await client.receive_json()
assert response["event"] == {
@ -1818,21 +1915,9 @@ async def test_restore_progress_after_restart(
) -> None:
"""Test restore backup progress after restart."""
supervisor_client.jobs.get_job.return_value = supervisor_jobs.Job(
name="backup_manager_partial_backup",
reference="1ef41507",
uuid=UUID("d17bd02be1f0437fa7264b16d38f700e"),
progress=0.0,
stage="copy_additional_locations",
done=True,
errors=[],
child_jobs=[],
)
supervisor_client.jobs.get_job.return_value = TEST_JOB_DONE
with patch.dict(
os.environ,
MOCK_ENVIRON | {RESTORE_JOB_ID_ENV: "d17bd02be1f0437fa7264b16d38f700e"},
):
with patch.dict(os.environ, MOCK_ENVIRON | {RESTORE_JOB_ID_ENV: TEST_JOB_ID}):
assert await async_setup_component(hass, BACKUP_DOMAIN, {BACKUP_DOMAIN: {}})
client = await hass_ws_client(hass)
@ -1860,10 +1945,7 @@ async def test_restore_progress_after_restart_unknown_job(
supervisor_client.jobs.get_job.side_effect = SupervisorError
with patch.dict(
os.environ,
MOCK_ENVIRON | {RESTORE_JOB_ID_ENV: "d17bd02be1f0437fa7264b16d38f700e"},
):
with patch.dict(os.environ, MOCK_ENVIRON | {RESTORE_JOB_ID_ENV: TEST_JOB_ID}):
assert await async_setup_component(hass, BACKUP_DOMAIN, {BACKUP_DOMAIN: {}})
client = await hass_ws_client(hass)