Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion hydra_base/lib/attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1180,7 +1180,8 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm=
ra_j.attr = JSONObject(ra.attr)
network_attributes.append(ra_j)

cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60)
if return_orm is not True:
cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60)

return network_attributes

Expand Down
11 changes: 7 additions & 4 deletions hydra_base/lib/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import logging
import datetime
from hydra_base import config as hydraconfig
from hydra_base.lib.objects import JSONObject, Dataset
import tempfile

log = logging.getLogger(__name__)
Expand All @@ -15,8 +16,9 @@
def _init_diskcache():
log.info("Using diskcache for caching.")
global cache
import diskcache as dc
cache = dc.Cache(tempfile.gettempdir())
import diskcache_community as dc
cache = dc.Cache(tempfile.gettempdir(),
disk=dc.SafeDisk.with_types(JSONObject, Dataset).with_options(on_error="miss"))

class _MemcachedWithFallback:
"""Wraps a pymemcache client and falls back to diskcache on connection errors."""
Expand Down Expand Up @@ -61,7 +63,7 @@ def flush_all(self):
try:
from pymemcache.client.base import Client as MemcacheClient
from pymemcache.serde import pickle_serde
import diskcache as dc
import diskcache_community as dc

host = hydraconfig.get('cache', 'host', '127.0.0.1')
port = hydraconfig.get('cache', 'port', 11211)
Expand All @@ -77,7 +79,8 @@ def flush_all(self):
except Exception:
raise ConnectionError("Memcached server not responding.")

_fallback = dc.Cache(tempfile.gettempdir())
_fallback = dc.Cache(tempfile.gettempdir(),
disk=dc.SafeDisk.with_types(JSONObject, Dataset).with_options(on_error="miss"))
cache = _MemcachedWithFallback(_mc, _fallback)

except (ModuleNotFoundError, ConnectionError) as e:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies=[
"cheroot",
"beaker",
"click",
"diskcache",
"diskcache-community>=5.8.4",
"fsspec",
"h5py",
"lxml",
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ cheroot
beaker
cheroot
click
diskcache
diskcache-community>=5.8.4
fsspec
h5py
lxml
Expand Down
27 changes: 27 additions & 0 deletions tests/templates/test_template_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import tempfile
import time
import pytest
from diskcache_community import Cache, Disk

import hydra_base as hb
from hydra_base.lib.objects import JSONObject
Expand Down Expand Up @@ -112,6 +113,19 @@ def _cached_entry(template_id):
return cache.get(f"{CACHE_KEY}_{template_id}")


planted_calls = []


def _planted():
planted_calls.append(True)
return JSONObject({'name': 'planted', 'templatetypes': []})


class _Planted:
def __reduce__(self):
return (_planted, ())


# ---------------------------------------------------------------------------
# tests
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -319,3 +333,16 @@ def test_import_template_dict_cache_keeps_empty_typeattrs(self, db):
assert list(rebuilt_types["EmptyEdge"].typeattrs) == []
assert len(first_types["TypeWithAttr"].typeattrs) == len(rebuilt_types["TypeWithAttr"].typeattrs) == 1

def test_planted_pickle_is_not_loaded(self, db):
attr = _create_attr()
new_tmpl = template_lib.add_template(
_make_template_input(attr.id), user_id=USER_ID
)
with Cache(cache.directory, disk=Disk) as other_writer:
other_writer.set(f"{CACHE_KEY}_{new_tmpl.id}", _Planted())

result = template_lib.get_template(new_tmpl.id, user_id=USER_ID)

assert planted_calls == []
assert result.name == new_tmpl.name
assert len(result.templatetypes) == 1