From c3f7a45d61e418b18668f1ac95676a21fc340c69 Mon Sep 17 00:00:00 2001 From: Mike Degatano Date: Wed, 17 Aug 2022 16:50:08 -0400 Subject: [PATCH] Fix memory calculation for cgroupv2 (#3802) --- supervisor/docker/stats.py | 9 +- tests/docker/test_stats.py | 40 +++++-- tests/fixtures/container_stats.json | 7 +- tests/fixtures/container_stats_cgroupv1.json | 120 +++++++++++++++++++ tests/fixtures/container_stats_cgroupv2.json | 99 +++++++++++++++ 5 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/container_stats_cgroupv1.json create mode 100644 tests/fixtures/container_stats_cgroupv2.json diff --git a/supervisor/docker/stats.py b/supervisor/docker/stats.py index 49a9e62dd..b82aecea2 100644 --- a/supervisor/docker/stats.py +++ b/supervisor/docker/stats.py @@ -14,11 +14,18 @@ class DockerStats: self._blk_write = 0 try: + # cgroupv1 & Docker > 19.03 if "total_inactive_file" in stats["memory_stats"]["stats"]: cache = stats["memory_stats"]["stats"]["total_inactive_file"] - else: + + # Docker <= 19.03 + elif "cache" in stats["memory_stats"]["stats"]: cache = stats["memory_stats"]["stats"]["cache"] + # cgroupv2 + else: + cache = stats["memory_stats"]["stats"]["inactive_file"] + self._memory_usage = stats["memory_stats"]["usage"] - cache self._memory_limit = stats["memory_stats"]["limit"] except KeyError: diff --git a/tests/docker/test_stats.py b/tests/docker/test_stats.py index 02ffe5423..e0dca4a58 100644 --- a/tests/docker/test_stats.py +++ b/tests/docker/test_stats.py @@ -1,20 +1,40 @@ """Test docker stats.""" +import pytest + from supervisor.docker.stats import DockerStats from tests.common import load_json_fixture -def test_cpu_presentage(docker): - """Test CPU presentage.""" - stats_fixtrue = load_json_fixture("container_stats.json") - stats = DockerStats(stats_fixtrue) +@pytest.mark.parametrize( + "stats_mock", + ["container_stats", "container_stats_cgroupv1", "container_stats_cgroupv2"], +) +def test_cpu_percentage(stats_mock: str): + """Test CPU percentage.""" + stats_fixture = load_json_fixture(f"{stats_mock}.json") + stats = DockerStats(stats_fixture) - stats._calc_cpu_percent(stats_fixtrue) # pylint: disable=protected-access + stats._calc_cpu_percent(stats_fixture) # pylint: disable=protected-access assert stats.cpu_percent == 90.0 - stats_fixtrue["cpu_stats"]["cpu_usage"]["total_usage"] = 0 - stats_fixtrue["precpu_stats"]["cpu_usage"]["total_usage"] = 0 - stats_fixtrue["cpu_stats"]["system_cpu_usage"] = 0 - stats_fixtrue["precpu_stats"]["system_cpu_usage"] = 0 - stats._calc_cpu_percent(stats_fixtrue) # pylint: disable=protected-access + stats_fixture["cpu_stats"]["cpu_usage"]["total_usage"] = 0 + stats_fixture["precpu_stats"]["cpu_usage"]["total_usage"] = 0 + stats_fixture["cpu_stats"]["system_cpu_usage"] = 0 + stats_fixture["precpu_stats"]["system_cpu_usage"] = 0 + stats._calc_cpu_percent(stats_fixture) # pylint: disable=protected-access assert stats.cpu_percent == 0.0 + + +@pytest.mark.parametrize( + "stats_mock", + ["container_stats", "container_stats_cgroupv1", "container_stats_cgroupv2"], +) +async def test_memory_usage(stats_mock: str): + """Test memory usage calculation.""" + stats_fixture = load_json_fixture(f"{stats_mock}.json") + stats = DockerStats(stats_fixture) + + assert stats.memory_limit == 4000000000 + assert stats.memory_usage == 59700000 + assert stats.memory_percent == 1.49 diff --git a/tests/fixtures/container_stats.json b/tests/fixtures/container_stats.json index f7131bf21..c3d4242ce 100644 --- a/tests/fixtures/container_stats.json +++ b/tests/fixtures/container_stats.json @@ -14,7 +14,10 @@ "online_cpus": 24 }, "memory_stats": { - "usage": 250000000, - "limit": 330000000000 + "usage": 60000000, + "stats": { + "cache": 300000 + }, + "limit": 4000000000 } } diff --git a/tests/fixtures/container_stats_cgroupv1.json b/tests/fixtures/container_stats_cgroupv1.json new file mode 100644 index 000000000..6c73548cf --- /dev/null +++ b/tests/fixtures/container_stats_cgroupv1.json @@ -0,0 +1,120 @@ +{ + "read": "2022-08-17T14:39:31.182030268Z", + "preread": "2022-08-17T14:39:30.176446102Z", + "pids_stats": { "current": 87 }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { "major": 179, "minor": 0, "op": "Read", "value": 555053056 }, + { "major": 179, "minor": 0, "op": "Write", "value": 9285632 }, + { "major": 179, "minor": 0, "op": "Sync", "value": 563703808 }, + { "major": 179, "minor": 0, "op": "Async", "value": 634880 }, + { "major": 179, "minor": 0, "op": "Discard", "value": 0 }, + { "major": 179, "minor": 0, "op": "Total", "value": 564338688 } + ], + "io_serviced_recursive": [ + { "major": 179, "minor": 0, "op": "Read", "value": 32545 }, + { "major": 179, "minor": 0, "op": "Write", "value": 568 }, + { "major": 179, "minor": 0, "op": "Sync", "value": 33055 }, + { "major": 179, "minor": 0, "op": "Async", "value": 58 }, + { "major": 179, "minor": 0, "op": "Discard", "value": 0 }, + { "major": 179, "minor": 0, "op": "Total", "value": 33113 } + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": [] + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 190, + "percpu_usage": [ + 81992750178, 76745418076, 187526993366, 187725247855, 179490503539, + 187568181067 + ], + "usage_in_kernelmode": 117410000000, + "usage_in_usermode": 749400000000 + }, + "system_cpu_usage": 200, + "online_cpus": 6, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 100, + "percpu_usage": [ + 81992750178, 76744282619, 187526971782, 187725247855, 179490295207, + 187568079858 + ], + "usage_in_kernelmode": 117410000000, + "usage_in_usermode": 749400000000 + }, + "system_cpu_usage": 100, + "online_cpus": 6, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 60000000, + "max_usage": 574808064, + "stats": { + "active_anon": 31461376, + "active_file": 7278592, + "cache": 11374592, + "dirty": 0, + "hierarchical_memory_limit": 9223372036854771712, + "hierarchical_memsw_limit": 9223372036854771712, + "inactive_anon": 239869952, + "inactive_file": 4096000, + "mapped_file": 9691136, + "pgfault": 5491296, + "pgmajfault": 15245, + "pgpgin": 4522918, + "pgpgout": 4477915, + "rss": 271331328, + "rss_huge": 90177536, + "total_active_anon": 31461376, + "total_active_file": 7278592, + "total_cache": 11374592, + "total_dirty": 0, + "total_inactive_anon": 239869952, + "total_inactive_file": 300000, + "total_mapped_file": 9691136, + "total_pgfault": 5491296, + "total_pgmajfault": 15245, + "total_pgpgin": 4522918, + "total_pgpgout": 4477915, + "total_rss": 271331328, + "total_rss_huge": 90177536, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "limit": 4000000000 + }, + "name": "/addon_local_test", + "id": "24808489f0c07124b913c5ca5b8741c191b2ee6f7f3c3d0ec69ac9c9329bba8d", + "networks": { + "eth0": { + "rx_bytes": 686762770, + "rx_packets": 3323163, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 488473500, + "tx_packets": 73299, + "tx_errors": 0, + "tx_dropped": 0 + } + } +} diff --git a/tests/fixtures/container_stats_cgroupv2.json b/tests/fixtures/container_stats_cgroupv2.json new file mode 100644 index 000000000..181fafab0 --- /dev/null +++ b/tests/fixtures/container_stats_cgroupv2.json @@ -0,0 +1,99 @@ +{ + "read": "2022-08-17T14:39:53.107857116Z", + "preread": "2022-08-17T14:39:52.100845337Z", + "pids_stats": { "current": 29, "limit": 4234 }, + "blkio_stats": { + "io_service_bytes_recursive": [ + { "major": 179, "minor": 0, "op": "read", "value": 0 }, + { "major": 179, "minor": 0, "op": "write", "value": 413696 } + ], + "io_serviced_recursive": null, + "io_queue_recursive": null, + "io_service_time_recursive": null, + "io_wait_time_recursive": null, + "io_merged_recursive": null, + "io_time_recursive": null, + "sectors_recursive": null + }, + "num_procs": 0, + "storage_stats": {}, + "cpu_stats": { + "cpu_usage": { + "total_usage": 190, + "usage_in_kernelmode": 725181000, + "usage_in_usermode": 4916544000 + }, + "system_cpu_usage": 200, + "online_cpus": 6, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "precpu_stats": { + "cpu_usage": { + "total_usage": 100, + "usage_in_kernelmode": 725181000, + "usage_in_usermode": 4916544000 + }, + "system_cpu_usage": 100, + "online_cpus": 6, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "memory_stats": { + "usage": 60000000, + "stats": { + "active_anon": 32768, + "active_file": 290816, + "anon": 53714944, + "anon_thp": 4194304, + "file": 565248, + "file_dirty": 0, + "file_mapped": 0, + "file_writeback": 0, + "inactive_anon": 53682176, + "inactive_file": 300000, + "kernel_stack": 475136, + "pgactivate": 49, + "pgdeactivate": 0, + "pgfault": 90791, + "pglazyfree": 0, + "pglazyfreed": 0, + "pgmajfault": 0, + "pgrefill": 0, + "pgscan": 0, + "pgsteal": 0, + "shmem": 0, + "slab": 2539120, + "slab_reclaimable": 2106952, + "slab_unreclaimable": 432168, + "sock": 0, + "thp_collapse_alloc": 0, + "thp_fault_alloc": 4, + "unevictable": 0, + "workingset_activate": 0, + "workingset_nodereclaim": 0, + "workingset_refault": 0 + }, + "limit": 4000000000 + }, + "name": "/addon_local_test", + "id": "38cf89dc726cd0b74d205a044eb8de16db12d715b425cce5db3aef086e737836", + "networks": { + "eth0": { + "rx_bytes": 1243260, + "rx_packets": 6214, + "rx_errors": 0, + "rx_dropped": 0, + "tx_bytes": 5274, + "tx_packets": 50, + "tx_errors": 0, + "tx_dropped": 0 + } + } +}