diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 131cf270..0fe66671 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -1180,7 +1180,8 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm= ra_j.attr = JSONObject(ra.attr) network_attributes.append(ra_j) - cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60) + if return_orm is not True: + cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60) return network_attributes diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index e34dba86..11e4ca1b 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -7,6 +7,7 @@ import logging import datetime from hydra_base import config as hydraconfig +from hydra_base.lib.objects import JSONObject, Dataset import tempfile log = logging.getLogger(__name__) @@ -15,8 +16,9 @@ def _init_diskcache(): log.info("Using diskcache for caching.") global cache - import diskcache as dc - cache = dc.Cache(tempfile.gettempdir()) + import diskcache_community as dc + cache = dc.Cache(tempfile.gettempdir(), + disk=dc.SafeDisk.with_types(JSONObject, Dataset).with_options(on_error="miss")) class _MemcachedWithFallback: """Wraps a pymemcache client and falls back to diskcache on connection errors.""" @@ -61,7 +63,7 @@ def flush_all(self): try: from pymemcache.client.base import Client as MemcacheClient from pymemcache.serde import pickle_serde - import diskcache as dc + import diskcache_community as dc host = hydraconfig.get('cache', 'host', '127.0.0.1') port = hydraconfig.get('cache', 'port', 11211) @@ -77,7 +79,8 @@ def flush_all(self): except Exception: raise ConnectionError("Memcached server not responding.") - _fallback = dc.Cache(tempfile.gettempdir()) + _fallback = dc.Cache(tempfile.gettempdir(), + disk=dc.SafeDisk.with_types(JSONObject, Dataset).with_options(on_error="miss")) cache = _MemcachedWithFallback(_mc, _fallback) except (ModuleNotFoundError, ConnectionError) as e: diff --git a/pyproject.toml b/pyproject.toml index 0bd2137e..2039b675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies=[ "cheroot", "beaker", "click", - "diskcache", + "diskcache-community>=5.8.4", "fsspec", "h5py", "lxml", diff --git a/requirements.txt b/requirements.txt index 206d3010..07fbaafa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ cheroot beaker cheroot click -diskcache +diskcache-community>=5.8.4 fsspec h5py lxml diff --git a/tests/templates/test_template_cache.py b/tests/templates/test_template_cache.py index 054ee246..48390057 100644 --- a/tests/templates/test_template_cache.py +++ b/tests/templates/test_template_cache.py @@ -38,6 +38,7 @@ import tempfile import time import pytest +from diskcache_community import Cache, Disk import hydra_base as hb from hydra_base.lib.objects import JSONObject @@ -112,6 +113,19 @@ def _cached_entry(template_id): return cache.get(f"{CACHE_KEY}_{template_id}") +planted_calls = [] + + +def _planted(): + planted_calls.append(True) + return JSONObject({'name': 'planted', 'templatetypes': []}) + + +class _Planted: + def __reduce__(self): + return (_planted, ()) + + # --------------------------------------------------------------------------- # tests # --------------------------------------------------------------------------- @@ -319,3 +333,16 @@ def test_import_template_dict_cache_keeps_empty_typeattrs(self, db): assert list(rebuilt_types["EmptyEdge"].typeattrs) == [] assert len(first_types["TypeWithAttr"].typeattrs) == len(rebuilt_types["TypeWithAttr"].typeattrs) == 1 + def test_planted_pickle_is_not_loaded(self, db): + attr = _create_attr() + new_tmpl = template_lib.add_template( + _make_template_input(attr.id), user_id=USER_ID + ) + with Cache(cache.directory, disk=Disk) as other_writer: + other_writer.set(f"{CACHE_KEY}_{new_tmpl.id}", _Planted()) + + result = template_lib.get_template(new_tmpl.id, user_id=USER_ID) + + assert planted_calls == [] + assert result.name == new_tmpl.name + assert len(result.templatetypes) == 1