From f2c286bf17ef3c847ab6fb9d99b0e80f2128d74d Mon Sep 17 00:00:00 2001 From: Ahter Sonmez Date: Thu, 1 Jun 2023 13:36:00 +0100 Subject: [PATCH 1/4] Handle the creation of cache key suffixes Explanation will come here. --- django/utils/cache_key.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 django/utils/cache_key.py diff --git a/django/utils/cache_key.py b/django/utils/cache_key.py new file mode 100644 index 000000000000..8622a1dfb1e4 --- /dev/null +++ b/django/utils/cache_key.py @@ -0,0 +1,25 @@ +""" +Helpers for creating cache keys. + +The entry point is cache.py in this directory. +""" + + +def handle_cache_key_suffixes( + *, cache_key: str, language_code: str, timezone: str +) -> str: + """ + Handle the creation of the cache key suffixes for i18n and tz without the request object. + + TODO: This can be made more generic with a dict of + suffixes. + """ + # Add the locale. + if language_code: + cache_key += ".%s" % language_code + + # Add the timezone. + if timezone: + cache_key += ".%s" % timezone + + return cache_key From 78430404b97e89514e2f8c06201d96d6f59ac413 Mon Sep 17 00:00:00 2001 From: Ahter Sonmez Date: Thu, 1 Jun 2023 13:42:58 +0100 Subject: [PATCH 2/4] Generate the cache header key WIP in the explanation. --- django/utils/cache_key.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/django/utils/cache_key.py b/django/utils/cache_key.py index 8622a1dfb1e4..3082738392fe 100644 --- a/django/utils/cache_key.py +++ b/django/utils/cache_key.py @@ -3,6 +3,7 @@ The entry point is cache.py in this directory. """ +from django.utils.crypto import md5 def handle_cache_key_suffixes( @@ -23,3 +24,18 @@ def handle_cache_key_suffixes( cache_key += ".%s" % timezone return cache_key + + +def generate_cache_header_key(key_prefix, key_suffix, uri): + """ + Return a cache key for the header cache. + + This make it possible to get the header key without + the request object as well. + """ + url_hash = md5(uri, usedforsecurity=False) + cache_key = "views.decorators.cache.cache_header.%s.%s" % ( + key_prefix, + url_hash.hexdigest(), + ) + return WIP_TO_BE_DECIDED(cache_key) From 1320ee1a027e796539eb74f0ddc6d645255b5ef7 Mon Sep 17 00:00:00 2001 From: Ahter Sonmez Date: Thu, 1 Jun 2023 14:01:28 +0100 Subject: [PATCH 3/4] Add method to invalidate cache --- django/utils/cache.py | 95 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/django/utils/cache.py b/django/utils/cache.py index 90292ce4da60..193be81cef18 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -440,3 +440,98 @@ def _to_tuple(s): if len(t) == 2: return t[0].lower(), t[1] return t[0].lower(), True + + +def clear_cache_for_path( + path, key_prefix=None, cache=None, url_scheme="http", http_host="www.example.com" +): + """ + WIP! WIP! WIP! + + Clear the cache(s) for a given path, in a 'request-agnostic' way. + + This helper method is a different entry point to cache clearing, aimed to be triggered by different views, + signals or management commands. + + :param path: + The path to clear the cache for. + + :param key_prefix: + The key prefix to use. If not given, the default CACHE_MIDDLEWARE_KEY_PREFIX is used. + + :param cache: + The cache type to use. If not given, the default CACHE_MIDDLEWARE_ALIAS is used. + + :param url_scheme: + The URL scheme to use. Defaults to 'http'. + + :param http_host: + The HTTP host to use. Defaults to 'www.example.com'. + + If the path is not in the cache, do nothing. + + Clears cache for: + - hashes + - pages + - headers + + TODO: Ideally, we shouldn't have to pass in the url_scheme and http_host, but this is a WIP for now. + """ + if key_prefix is None: + key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX + + if cache is None: + cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] + + # Construct the cache key suffix. + full_path = f"{url_scheme}://{http_host}{path}" + absolute_uri = iri_to_uri(full_path) + absolute_uri_hash = md5(absolute_uri.encode("ascii"), usedforsecurity=False) + key_suffix = absolute_uri_hash.hexdigest() + # maybe a better name? + + hashes_key = _get_cache_hash_key(key_prefix, key_suffix) + # not_working_hashes_ignore = cache.get(hashes_key, None) + hashes = cache._cache.get(hashes_key, None) + # should we access private API? + + cache.delete_pattern("%s*" % key_prefix) + + if hashes: + for page_hash in hashes: + page_key = _get_page_cache_key(key_prefix, suffix, page_hash) + cache.delete(page_key) + + # Delete the header cache. + header_key = _get_header_cache_key(key_prefix, key_suffix) + cache.delete(header_key) + + # Delete the page cache. + get_request_page_key = _get_page_cache_key(key_prefix, key_suffix, "GET") + cache.delete(get_request_page_key) + + head_request_page_key = _get_page_cache_key(key_prefix, key_suffix, "HEAD") + cache.delete(head_request_page_key) + + # Delete the hashes key? Really? What's this? + cache.delete(hashes_key) + + +def _get_cache_hash_key(key_prefix, suffix): + return f"{DJANGO_CACHE_MODULE}.cache_hashes.{key_prefix}.{suffix}" + + +def _get_page_cache_key(key_prefix, suffix, method): + """ + This is for HEAD and GET requests. + + Maybe implement starts with? + """ + ctx = md5(usedforsecurity=False) + page_hash_hex = ctx.hexdigest() + + return f"{DJANGO_CACHE_MODULE}.cache_page.{key_prefix}.{method}.{suffix}.{page_hash_hex}" + + +def _get_header_cache_key(key_prefix, suffix): + return f"{DJANGO_CACHE_MODULE}.cache_header.{key_prefix}.{suffix}" \ No newline at end of file From 2eccd657574d52ed24f990f74c8e6da86f50494a Mon Sep 17 00:00:00 2001 From: Ahter Sonmez Date: Sun, 23 Jul 2023 10:03:16 +0100 Subject: [PATCH 4/4] WIP --- django/utils/cache.py | 148 +++++++++++--------------------------- django/utils/cache_key.py | 41 ----------- 2 files changed, 43 insertions(+), 146 deletions(-) delete mode 100644 django/utils/cache_key.py diff --git a/django/utils/cache.py b/django/utils/cache.py index 193be81cef18..5a5df2d9de84 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -23,6 +23,7 @@ from django.core.cache import caches from django.http import HttpResponse, HttpResponseNotModified from django.utils.crypto import md5 +from django.utils.encoding import iri_to_uri from django.utils.http import http_date, parse_etags, parse_http_date_safe, quote_etag from django.utils.log import log_response from django.utils.regex_helper import _lazy_re_compile @@ -333,20 +334,35 @@ def has_vary_header(response, header_query): return header_query.lower() in existing_headers -def _i18n_cache_key_suffix(request, cache_key): - """If necessary, add the current locale or time zone to the cache key.""" +def _i18n_cache_key_suffix(request, cache_key:str) -> str: + """ + If necessary, add the current locale or time zone to the cache key. + """ + language_code = "" + timezone = "" + if settings.USE_I18N: # first check if LocaleMiddleware or another middleware added # LANGUAGE_CODE to request, then fall back to the active language # which in turn can also fall back to settings.LANGUAGE_CODE - cache_key += ".%s" % getattr(request, "LANGUAGE_CODE", get_language()) + language_code = getattr(request, "LANGUAGE_CODE", get_language()) if settings.USE_TZ: - cache_key += ".%s" % get_current_timezone_name() - return cache_key + timezone = get_current_timezone_name() + + return cache_key.handle_cache_key_suffixes( + cache_key=cache_key, language_code=language_code, timezone=timezone + ) + + + def _generate_cache_key(request, method, headerlist, key_prefix): - """Return a cache key from the headers given in the header list.""" + """ + Entry point for cache key generation. + + Return a cache key from the headers given in the header list. + """ ctx = md5(usedforsecurity=False) for header in headerlist: value = request.META.get(header) @@ -359,19 +375,32 @@ def _generate_cache_key(request, method, headerlist, key_prefix): url.hexdigest(), ctx.hexdigest(), ) + return _i18n_cache_key_suffix(request, cache_key) -def _generate_cache_header_key(key_prefix, request): - """Return a cache key for the header cache.""" - url = md5(request.build_absolute_uri().encode("ascii"), usedforsecurity=False) +def _generate_cache_key_from_request(*, key_prefix, request): + uri = _get_uri_from_request(request) + url_hash = md5(uri, usedforsecurity=False) cache_key = "views.decorators.cache.cache_header.%s.%s" % ( key_prefix, - url.hexdigest(), + url_hash.hexdigest(), ) return _i18n_cache_key_suffix(request, cache_key) +def _generate_cache_key_from_uri(*, key_prefix, uri): + + url_hash = md5(uri, usedforsecurity=False) + cache_key = "views.decorators.cache.cache_header.%s.%s" % ( + key_prefix, + url_hash.hexdigest(), + ) + return _i18n_cache_key_suffix(cache_key) + + + + def get_cache_key(request, key_prefix=None, method="GET", cache=None): """ Return a cache key based on the request URL and query. It can be used @@ -411,6 +440,8 @@ def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cach key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX if cache_timeout is None: cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS + + # Generate the cache key. cache_key = _generate_cache_header_key(key_prefix, request) if cache is None: cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] @@ -426,6 +457,8 @@ def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cach if header != "ACCEPT_LANGUAGE" or not is_accept_language_redundant: headerlist.append("HTTP_" + header) headerlist.sort() + + # Set the cache key. cache.set(cache_key, headerlist, cache_timeout) return _generate_cache_key(request, request.method, headerlist, key_prefix) else: @@ -440,98 +473,3 @@ def _to_tuple(s): if len(t) == 2: return t[0].lower(), t[1] return t[0].lower(), True - - -def clear_cache_for_path( - path, key_prefix=None, cache=None, url_scheme="http", http_host="www.example.com" -): - """ - WIP! WIP! WIP! - - Clear the cache(s) for a given path, in a 'request-agnostic' way. - - This helper method is a different entry point to cache clearing, aimed to be triggered by different views, - signals or management commands. - - :param path: - The path to clear the cache for. - - :param key_prefix: - The key prefix to use. If not given, the default CACHE_MIDDLEWARE_KEY_PREFIX is used. - - :param cache: - The cache type to use. If not given, the default CACHE_MIDDLEWARE_ALIAS is used. - - :param url_scheme: - The URL scheme to use. Defaults to 'http'. - - :param http_host: - The HTTP host to use. Defaults to 'www.example.com'. - - If the path is not in the cache, do nothing. - - Clears cache for: - - hashes - - pages - - headers - - TODO: Ideally, we shouldn't have to pass in the url_scheme and http_host, but this is a WIP for now. - """ - if key_prefix is None: - key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX - - if cache is None: - cache = caches[settings.CACHE_MIDDLEWARE_ALIAS] - - # Construct the cache key suffix. - full_path = f"{url_scheme}://{http_host}{path}" - absolute_uri = iri_to_uri(full_path) - absolute_uri_hash = md5(absolute_uri.encode("ascii"), usedforsecurity=False) - key_suffix = absolute_uri_hash.hexdigest() - # maybe a better name? - - hashes_key = _get_cache_hash_key(key_prefix, key_suffix) - # not_working_hashes_ignore = cache.get(hashes_key, None) - hashes = cache._cache.get(hashes_key, None) - # should we access private API? - - cache.delete_pattern("%s*" % key_prefix) - - if hashes: - for page_hash in hashes: - page_key = _get_page_cache_key(key_prefix, suffix, page_hash) - cache.delete(page_key) - - # Delete the header cache. - header_key = _get_header_cache_key(key_prefix, key_suffix) - cache.delete(header_key) - - # Delete the page cache. - get_request_page_key = _get_page_cache_key(key_prefix, key_suffix, "GET") - cache.delete(get_request_page_key) - - head_request_page_key = _get_page_cache_key(key_prefix, key_suffix, "HEAD") - cache.delete(head_request_page_key) - - # Delete the hashes key? Really? What's this? - cache.delete(hashes_key) - - -def _get_cache_hash_key(key_prefix, suffix): - return f"{DJANGO_CACHE_MODULE}.cache_hashes.{key_prefix}.{suffix}" - - -def _get_page_cache_key(key_prefix, suffix, method): - """ - This is for HEAD and GET requests. - - Maybe implement starts with? - """ - ctx = md5(usedforsecurity=False) - page_hash_hex = ctx.hexdigest() - - return f"{DJANGO_CACHE_MODULE}.cache_page.{key_prefix}.{method}.{suffix}.{page_hash_hex}" - - -def _get_header_cache_key(key_prefix, suffix): - return f"{DJANGO_CACHE_MODULE}.cache_header.{key_prefix}.{suffix}" \ No newline at end of file diff --git a/django/utils/cache_key.py b/django/utils/cache_key.py deleted file mode 100644 index 3082738392fe..000000000000 --- a/django/utils/cache_key.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -Helpers for creating cache keys. - -The entry point is cache.py in this directory. -""" -from django.utils.crypto import md5 - - -def handle_cache_key_suffixes( - *, cache_key: str, language_code: str, timezone: str -) -> str: - """ - Handle the creation of the cache key suffixes for i18n and tz without the request object. - - TODO: This can be made more generic with a dict of - suffixes. - """ - # Add the locale. - if language_code: - cache_key += ".%s" % language_code - - # Add the timezone. - if timezone: - cache_key += ".%s" % timezone - - return cache_key - - -def generate_cache_header_key(key_prefix, key_suffix, uri): - """ - Return a cache key for the header cache. - - This make it possible to get the header key without - the request object as well. - """ - url_hash = md5(uri, usedforsecurity=False) - cache_key = "views.decorators.cache.cache_header.%s.%s" % ( - key_prefix, - url_hash.hexdigest(), - ) - return WIP_TO_BE_DECIDED(cache_key)