Adding additional unit tests

test_app/tests/lib/cache/test_fallback_cache.py

@@ -0,0 +1,226 @@
+import logging
+import tempfile
+import time
+from pathlib import Path
+from unittest import mock
+
+import pytest
+from django.core import cache as django_cache
+from django.core.cache.backends.base import BaseCache
+from django.test import override_settings
+
+from ansible_base.lib.cache.fallback_cache import FALLBACK_CACHE, PRIMARY_CACHE, DABCacheWithFallback
+
+
+class BreakableCache(BaseCache):
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(BreakableCache, cls).__new__(cls)
+            cls.__initialized = False
+        return cls._instance
+
+    def __init__(self, location, params):
+        if self.__initialized:
+            return
+        self.cache = {}
+        options = params.get("OPTIONS", {})
+        self.working = options.get("working", True)
+        self.__initialized = True
+
+    def add(self, key, value, timeout=300, version=None):
+        self.cache[key] = value
+
+    def get(self, key, default=None, version=None):
+        if self.working:
+            return self.cache.get(key, default)
+        else:
+            raise RuntimeError(f"Sorry, cache no worky {self}")
+
+    def set(self, key, value, timeout=300, version=None, client=None):
+        self.cache[key] = value
+
+    def delete(self, key, version=None):
+        self.cache.pop(key, None)
+
+    def clear(self):
+        self.cache = {}
+
+    def breakit(self):
+        self.working = False
+
+    def fixit(self):
+        self.working = True
+
+
+cache_settings = {
+    'default': {
+        'BACKEND': 'ansible_base.lib.cache.fallback_cache.DABCacheWithFallback',
+    },
+    'primary': {
+        'BACKEND': 'test_app.tests.lib.cache.test_fallback_cache.BreakableCache',
+        'LOCATION': 'primary',
+    },
+    'fallback': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'fallback',
+    },
+}
+
+
+@override_settings(CACHES=cache_settings)
+def test_fallback_cache():
+    cache = django_cache.caches.create_connection('default')
+
+    primary = cache._primary_cache
+    fallback = cache._fallback_cache
+    cache.set('key', 'val1')
+    assert primary.get('key') == 'val1'
+    assert fallback.get('key') is None
+
+    primary.set('tobecleared', True)
+    primary.breakit()
+
+    # Breaks primary
+    cache.get('key')
+
+    # Sets in fallback
+    cache.set('key', 'val2')
+
+    assert cache.get('key', 'val2')
+
+    assert cache.get_active_cache() == FALLBACK_CACHE
+
+    primary.fixit()
+
+    # Check until primary is back
+    timeout = time.time() + 30
+    while True:
+        if cache.get_active_cache() == PRIMARY_CACHE:
+            break
+        if time.time() > timeout:
+            assert False
+        time.sleep(1)
+
+    # Ensure caches were cleared
+    assert cache.get('key') is None
+    assert fallback.get('key') is None
+    assert cache.get('tobecleared') is None
+
+    cache.set('key2', 'val3')
+
+    assert cache.get('key2') == 'val3'
+
+
+@override_settings(CACHES=cache_settings)
+def test_dead_primary():
+    primary_cache = django_cache.caches.create_connection('primary')
+    primary_cache.breakit()
+
+    # Kill post-shutdown logging from unfinished recovery checker
+    logging.getLogger('ansible_base.cache.fallback_cache').setLevel(logging.CRITICAL)
+
+    cache = django_cache.caches.create_connection('default')
+
+    cache.set('key', 'val')
+    cache.get('key')
+
+    # Check until fallback is set
+    timeout = time.time() + 30
+    while True:
+        if cache.get_active_cache() == FALLBACK_CACHE:
+            break
+        if time.time() > timeout:
+            assert False
+        time.sleep(1)
+
+
+@override_settings(CACHES=cache_settings)
+def test_ensure_temp_file_is_removed_on_init():
+    temp_file = Path(tempfile.NamedTemporaryFile().name)
+    with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file):
+        temp_file.touch()
+        DABCacheWithFallback(None, {})
+        assert temp_file.exists() is False
+
+
+@override_settings(CACHES=cache_settings)
+def test_ensure_initialization_wont_happen_twice():
+    with mock.patch('ansible_base.lib.cache.fallback_cache.ThreadPoolExecutor') as tfe:
+        cache = DABCacheWithFallback(None, {})
+        tfe.assert_called_once()
+        cache.__init__(None, {})
+        # when calling init again ThreadPoolExecute should not be called again so we should still have only one call
+        tfe.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    "method",
+    [
+        ('clear'),
+        ('delete'),
+        ('set'),
+        ('get'),
+        ('add'),
+    ],
+)
+@override_settings(CACHES=cache_settings)
+def test_all_methods_are_overwritten(method):
+    with mock.patch('ansible_base.lib.cache.fallback_cache.DABCacheWithFallback._op_with_fallback') as owf:
+        cache = DABCacheWithFallback(None, {})
+        if method == 'clear':
+            getattr(cache, method)()
+        elif method in ['delete', 'get']:
+            getattr(cache, method)('test_value')
+        else:
+            getattr(cache, method)('test_value', 1)
+        owf.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    "file_exists",
+    [
+        (True),
+        (False),
+    ],
+)
+@override_settings(CACHES=cache_settings)
+def test_check_primary_cache(file_exists):
+    temp_file = Path(tempfile.NamedTemporaryFile().name)
+    with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file):
+        # Initialization of the cache will clear the temp file so do this first
+        cache = DABCacheWithFallback(None, {})
+
+        # Create the temp file if needed
+        if file_exists:
+            temp_file.touch()
+        else:
+            try:
+                temp_file.unlink()
+            except Exception:
+                pass
+
+        mocked_function = mock.MagicMock(return_value=None)
+        cache._primary_cache.clear = mocked_function
+        cache.check_primary_cache()
+        if file_exists:
+            mocked_function.assert_called_once()
+        else:
+            mocked_function.assert_not_called()
+        assert temp_file.exists() is False
+
+
+@override_settings(CACHES=cache_settings)
+def test_file_unlink_exception_does_not_cause_failure():
+    temp_file = Path(tempfile.NamedTemporaryFile().name)
+    with mock.patch('ansible_base.lib.cache.fallback_cache._temp_file', temp_file):
+        cache = DABCacheWithFallback(None, {})
+        # We can't do: temp_file.unlink = mock.MagicMock(side_effect=Exception('failed to unlink exception'))
+        # Because unlink is marked as read only so we will just mock the cache.clear to raise in its place
+        mocked_function = mock.MagicMock(side_effect=Exception('failed to delete a file exception'))
+        cache._primary_cache.clear = mocked_function
+
+        temp_file.touch()
+        cache.check_primary_cache()
+        # No assertion needed because we just want to make sure check_primary_cache does not raise

test_app/tests/lib/dynamic_config/test_settings_logic.py

@@ -0,0 +1,23 @@
+import pytest
+
+from ansible_base.lib.dynamic_config.settings_logic import get_dab_settings
+
+
+@pytest.mark.parametrize(
+    "caches,expect_exception",
+    [
+        ({}, False),
+        ({"default": {"BACKEND": "junk"}}, False),
+        ({"default": {"BACKEND": "not_ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}}, True),
+        ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}}, True),
+        ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "primary": {}}, True),
+        ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "fallback": {}}, True),
+        ({"default": {"BACKEND": "ansible_base.lib.cache.fallback_cache.DABCacheWithFallback"}, "primary": {}, "fallback": {}}, False),
+    ],
+)
+def test_cache_settings(caches, expect_exception):
+    try:
+        get_dab_settings(installed_apps=[], caches=caches)
+    except RuntimeError:
+        if not expect_exception:
+            raise

test_app/tests/lib/redis/test_client.py

@@ -384,3 +384,15 @@ def test_redis_timeout():
         result = get_redis_status('redis://localhost', timeout=1)
         assert result['status'] == STATUS_FAILED
         assert result['exception'] == 'raised'
+
+
+def test_redis_standalone_removes_cluster_settings():
+    args = {'mode': 'standalone', 'cluster_error_retry_attempts': 4}
+    with mock.patch('redis.Redis.__init__', return_value=None) as rm:
+        from ansible_base.lib.redis.client import RedisClientGetter
+
+        client_getter = RedisClientGetter()
+        client_getter.get_client('rediss://localhost', **args)
+        rm.assert_called_once
+        assert 'host' in rm.call_args.kwargs
+        assert 'cluster_error_retry_attempts' not in rm.call_args.kwargs