From 95df850e7df5b0afb2d7e759582ada6c6fab0df6 Mon Sep 17 00:00:00 2001 From: Luke Meyer Date: Mon, 15 May 2017 16:09:43 -0400 Subject: disk/memory checks: make threshold configurable --- .../openshift_checks/disk_availability.py | 6 ++-- .../openshift_checks/memory_availability.py | 4 ++- .../test/disk_availability_test.py | 36 +++++++++++++++++++--- .../test/memory_availability_test.py | 31 ++++++++++++++++--- 4 files changed, 65 insertions(+), 12 deletions(-) (limited to 'roles') diff --git a/roles/openshift_health_checker/openshift_checks/disk_availability.py b/roles/openshift_health_checker/openshift_checks/disk_availability.py index c2792a0fe..962148cb8 100644 --- a/roles/openshift_health_checker/openshift_checks/disk_availability.py +++ b/roles/openshift_health_checker/openshift_checks/disk_availability.py @@ -27,10 +27,12 @@ class DiskAvailability(NotContainerizedMixin, OpenShiftCheck): def run(self, tmp, task_vars): group_names = get_var(task_vars, "group_names") ansible_mounts = get_var(task_vars, "ansible_mounts") - - min_free_bytes = max(self.recommended_disk_space_bytes.get(name, 0) for name in group_names) free_bytes = self.openshift_available_disk(ansible_mounts) + recommended_min = max(self.recommended_disk_space_bytes.get(name, 0) for name in group_names) + configured_min = int(get_var(task_vars, "openshift_check_min_host_disk_gb", default=0)) * 10**9 + min_free_bytes = configured_min or recommended_min + if free_bytes < min_free_bytes: return { 'failed': True, diff --git a/roles/openshift_health_checker/openshift_checks/memory_availability.py b/roles/openshift_health_checker/openshift_checks/memory_availability.py index 28805dc37..aed0e2d20 100644 --- a/roles/openshift_health_checker/openshift_checks/memory_availability.py +++ b/roles/openshift_health_checker/openshift_checks/memory_availability.py @@ -27,7 +27,9 @@ class MemoryAvailability(OpenShiftCheck): group_names = get_var(task_vars, "group_names") total_memory_bytes = get_var(task_vars, "ansible_memtotal_mb") * 10**6 - min_memory_bytes = max(self.recommended_memory_bytes.get(name, 0) for name in group_names) + recommended_min = max(self.recommended_memory_bytes.get(name, 0) for name in group_names) + configured_min = int(get_var(task_vars, "openshift_check_min_host_memory_gb", default=0)) * 10**9 + min_memory_bytes = configured_min or recommended_min if total_memory_bytes < min_memory_bytes: return { diff --git a/roles/openshift_health_checker/test/disk_availability_test.py b/roles/openshift_health_checker/test/disk_availability_test.py index 970b474d7..b353fa610 100644 --- a/roles/openshift_health_checker/test/disk_availability_test.py +++ b/roles/openshift_health_checker/test/disk_availability_test.py @@ -42,9 +42,10 @@ def test_cannot_determine_available_disk(ansible_mounts, extra_words): assert word in str(excinfo.value) -@pytest.mark.parametrize('group_names,ansible_mounts', [ +@pytest.mark.parametrize('group_names,configured_min,ansible_mounts', [ ( ['masters'], + 0, [{ 'mount': '/', 'size_available': 40 * 10**9 + 1, @@ -52,6 +53,7 @@ def test_cannot_determine_available_disk(ansible_mounts, extra_words): ), ( ['nodes'], + 0, [{ 'mount': '/', 'size_available': 15 * 10**9 + 1, @@ -59,6 +61,7 @@ def test_cannot_determine_available_disk(ansible_mounts, extra_words): ), ( ['etcd'], + 0, [{ 'mount': '/', 'size_available': 20 * 10**9 + 1, @@ -66,6 +69,15 @@ def test_cannot_determine_available_disk(ansible_mounts, extra_words): ), ( ['etcd'], + 1, # configure lower threshold + [{ + 'mount': '/', + 'size_available': 1 * 10**9 + 1, # way smaller than recommended + }], + ), + ( + ['etcd'], + 0, [{ # not enough space on / ... 'mount': '/', @@ -77,9 +89,10 @@ def test_cannot_determine_available_disk(ansible_mounts, extra_words): }], ), ]) -def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): +def test_succeeds_with_recommended_disk_space(group_names, configured_min, ansible_mounts): task_vars = dict( group_names=group_names, + openshift_check_min_host_disk_gb=configured_min, ansible_mounts=ansible_mounts, ) @@ -89,17 +102,28 @@ def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): assert not result.get('failed', False) -@pytest.mark.parametrize('group_names,ansible_mounts,extra_words', [ +@pytest.mark.parametrize('group_names,configured_min,ansible_mounts,extra_words', [ ( ['masters'], + 0, [{ 'mount': '/', 'size_available': 1, }], ['0.0 GB'], ), + ( + ['masters'], + 100, # set a higher threshold + [{ + 'mount': '/', + 'size_available': 50 * 10**9, # would normally be enough... + }], + ['100.0 GB'], + ), ( ['nodes'], + 0, [{ 'mount': '/', 'size_available': 1 * 10**9, @@ -108,6 +132,7 @@ def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): ), ( ['etcd'], + 0, [{ 'mount': '/', 'size_available': 1, @@ -116,6 +141,7 @@ def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): ), ( ['nodes', 'masters'], + 0, [{ 'mount': '/', # enough space for a node, not enough for a master @@ -125,6 +151,7 @@ def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): ), ( ['etcd'], + 0, [{ # enough space on / ... 'mount': '/', @@ -137,9 +164,10 @@ def test_succeeds_with_recommended_disk_space(group_names, ansible_mounts): ['0.0 GB'], ), ]) -def test_fails_with_insufficient_disk_space(group_names, ansible_mounts, extra_words): +def test_fails_with_insufficient_disk_space(group_names, configured_min, ansible_mounts, extra_words): task_vars = dict( group_names=group_names, + openshift_check_min_host_disk_gb=configured_min, ansible_mounts=ansible_mounts, ) diff --git a/roles/openshift_health_checker/test/memory_availability_test.py b/roles/openshift_health_checker/test/memory_availability_test.py index e161a5b9e..145169007 100644 --- a/roles/openshift_health_checker/test/memory_availability_test.py +++ b/roles/openshift_health_checker/test/memory_availability_test.py @@ -20,27 +20,37 @@ def test_is_active(group_names, is_active): assert MemoryAvailability.is_active(task_vars=task_vars) == is_active -@pytest.mark.parametrize('group_names,ansible_memtotal_mb', [ +@pytest.mark.parametrize('group_names,configured_min,ansible_memtotal_mb', [ ( ['masters'], + 0, 17200, ), ( ['nodes'], + 0, 8200, ), + ( + ['nodes'], + 1, # configure lower threshold + 2000, # too low for recommended but not for configured + ), ( ['etcd'], + 0, 22200, ), ( ['masters', 'nodes'], + 0, 17000, ), ]) -def test_succeeds_with_recommended_memory(group_names, ansible_memtotal_mb): +def test_succeeds_with_recommended_memory(group_names, configured_min, ansible_memtotal_mb): task_vars = dict( group_names=group_names, + openshift_check_min_host_memory_gb=configured_min, ansible_memtotal_mb=ansible_memtotal_mb, ) @@ -50,39 +60,50 @@ def test_succeeds_with_recommended_memory(group_names, ansible_memtotal_mb): assert not result.get('failed', False) -@pytest.mark.parametrize('group_names,ansible_memtotal_mb,extra_words', [ +@pytest.mark.parametrize('group_names,configured_min,ansible_memtotal_mb,extra_words', [ ( ['masters'], 0, + 0, ['0.0 GB'], ), ( ['nodes'], + 0, 100, ['0.1 GB'], ), + ( + ['nodes'], + 24, # configure higher threshold + 20000, # enough to meet recommended but not configured + ['20.0 GB'], + ), ( ['etcd'], + 0, -1, ['0.0 GB'], ), ( ['nodes', 'masters'], + 0, # enough memory for a node, not enough for a master 11000, ['11.0 GB'], ), ]) -def test_fails_with_insufficient_memory(group_names, ansible_memtotal_mb, extra_words): +def test_fails_with_insufficient_memory(group_names, configured_min, ansible_memtotal_mb, extra_words): task_vars = dict( group_names=group_names, + openshift_check_min_host_memory_gb=configured_min, ansible_memtotal_mb=ansible_memtotal_mb, ) check = MemoryAvailability(execute_module=fake_execute_module) result = check.run(tmp=None, task_vars=task_vars) - assert result['failed'] + assert result.get('failed', False) for word in 'below recommended'.split() + extra_words: assert word in result['msg'] -- cgit v1.2.3