diff --git a/build.py b/build.py
index 60255c1a..f9988e72 100644
--- a/build.py
+++ b/build.py
@@ -96,7 +96,7 @@ def release():
settings_path, next_version + snapshot_suffix,
'Bump version for next development iteration'
)
- git('push', '-u', 'origin', 'master')
+ git('push', '-u', 'origin', 'main')
try:
git('push', 'origin', release_tag)
try:
@@ -106,7 +106,7 @@ def release():
' git pull\n'
' git checkout %s\n'
' python build.py release\n'
- ' git checkout master\n\n'
+ ' git checkout main\n\n'
'on the other OSs now, then come back here and do:'
'\n\n'
' python build.py post_release\n'
@@ -117,7 +117,7 @@ def release():
raise
except:
git('revert', '--no-edit', revision_before + '..HEAD' )
- git('push', '-u', 'origin', 'master')
+ git('push', '-u', 'origin', 'main')
revision_before = git('rev-parse', 'HEAD').rstrip()
raise
except:
@@ -149,7 +149,7 @@ def post_release():
create_cloudfront_invalidation(cloudfront_items_to_invalidate)
record_release_on_server()
upload_core_to_github()
- git('checkout', 'master')
+ git('checkout', 'main')
def _prompt_for_next_version(release_version):
next_version = _get_suggested_next_version(release_version)
diff --git a/src/main/python/fman/__init__.py b/src/main/python/fman/__init__.py
index ab1f58d1..6470afaa 100644
--- a/src/main/python/fman/__init__.py
+++ b/src/main/python/fman/__init__.py
@@ -6,6 +6,7 @@
from os import getenv
from os.path import join, expanduser
from PyQt5.QtWidgets import QMessageBox
+from threading import local as _thread_local
import re
@@ -36,11 +37,16 @@
PLATFORM = platform.name()
if PLATFORM == 'Windows':
- DATA_DIRECTORY = join(getenv('APPDATA'), 'fman')
+ _appdata = getenv('APPDATA')
+ if not _appdata:
+ raise RuntimeError('APPDATA environment variable is not set')
+ DATA_DIRECTORY = join(_appdata, 'fman')
elif PLATFORM == 'Mac':
DATA_DIRECTORY = expanduser('~/Library/Application Support/fman')
elif PLATFORM == 'Linux':
DATA_DIRECTORY = expanduser('~/.config/fman')
+else:
+ raise NotImplementedError('Unsupported platform: %s' % PLATFORM)
class ApplicationCommand:
def __init__(self, window):
@@ -55,7 +61,7 @@ def aliases(self):
def _set_path_onerror(e, url):
if isinstance(e, FileNotFoundError):
return dirname(url)
- raise
+ raise e
class DirectoryPane:
def __init__(self, window, widget, command_registry):
@@ -63,12 +69,12 @@ def __init__(self, window, widget, command_registry):
self._widget = widget
self._command_registry = command_registry
self._listeners = []
- self._get_file_under_cursor_orig = self.get_file_under_cursor
+ self._file_under_cursor_override = _thread_local()
def _add_listener(self, listener):
self._listeners.append(listener)
def _broadcast(self, event, *args):
- for listener in self._listeners:
+ for listener in list(self._listeners):
getattr(listener, event)(*args)
def get_commands(self):
@@ -77,7 +83,7 @@ def run_command(self, name, args=None):
if args is None:
args = {}
while True:
- for listener in self._listeners:
+ for listener in list(self._listeners):
rewritten = listener.on_command(name, args)
if rewritten:
name, args = rewritten
@@ -98,6 +104,9 @@ def _remove_filter(self, filter_):
def get_selected_files(self):
return self._widget.get_selected_files()
def get_file_under_cursor(self):
+ override = getattr(self._file_under_cursor_override, 'value', None)
+ if override is not None:
+ return override
return self._widget.get_file_under_cursor()
def move_cursor_down(self, toggle_selection=False):
self._widget.move_cursor_down(toggle_selection)
@@ -113,14 +122,12 @@ def move_cursor_page_up(self, toggle_selection=False):
self._widget.move_cursor_page_up(toggle_selection)
def place_cursor_at(self, file_url):
self._widget.place_cursor_at(file_url)
- # TODO: Rename to get_location()
def get_path(self):
return self._widget.get_location()
- # TODO: Rename to set_location(...)
def set_path(self, dir_url, callback=None, onerror=_set_path_onerror):
args = dir_url, '', True
while True:
- for listener in self._listeners:
+ for listener in list(self._listeners):
rewritten = listener.before_location_change(*args)
if rewritten and rewritten != args:
args = rewritten
@@ -154,9 +161,11 @@ def _has_focus(self):
return self._widget.hasFocus()
@contextmanager
def _override_file_under_cursor(self, value):
- self.get_file_under_cursor = lambda: value
- yield
- self.get_file_under_cursor = self._get_file_under_cursor_orig
+ self._file_under_cursor_override.value = value
+ try:
+ yield
+ finally:
+ self._file_under_cursor_override.value = None
class Window:
def __init__(self, widget, panecmd_registry):
@@ -197,10 +206,10 @@ def on_doubleclicked(self, file_url):
pass
def on_name_edited(self, file_url, new_name):
pass
- # TODO: Rename to after_location_change()
def on_path_changed(self):
pass
def before_location_change(self, url, sort_column='', ascending=True):
+ """Return (url, sort_column, ascending) to rewrite, or None to allow."""
pass
def on_files_dropped(self, file_urls, dest_dir, is_copy_not_move):
pass
@@ -273,7 +282,7 @@ def unload_plugin(plugin_path):
class Task:
- class Canceled(KeyboardInterrupt):
+ class Canceled(Exception):
pass
def __init__(self, title, size=0, fn=lambda: None, args=(), kwargs=None):
diff --git a/src/main/python/fman/fs.py b/src/main/python/fman/fs.py
index 2cf8f00f..b6ee0308 100644
--- a/src/main/python/fman/fs.py
+++ b/src/main/python/fman/fs.py
@@ -67,6 +67,9 @@ def notify_file_removed(url):
_get_mother_fs().notify_file_removed(url)
class FileSystem:
+ """Base class for file system plugins. Set scheme to eg. 'ftp://'.
+ Methods receive scheme-stripped paths, except copy/move which get full URLs.
+ """
scheme = ''
@@ -108,7 +111,9 @@ def notify_file_added(self, path):
def notify_file_removed(self, path):
self._file_removed.trigger(self.scheme + path)
def notify_file_changed(self, path):
- for callback in self._file_changed_callbacks.get(path, []):
+ with self._file_changed_callbacks_lock:
+ callbacks = list(self._file_changed_callbacks.get(path, []))
+ for callback in callbacks:
callback(self.scheme + path)
def samefile(self, path1, path2):
return self.resolve(path1) == self.resolve(path2)
@@ -146,11 +151,12 @@ def prepare_trash(self, path):
raise self._operation_not_implemented()
return [Task(
'Deleting ' + path.rsplit('/', 1)[-1],
- fn=self.delete, args=(path,), size=1
+ fn=self.move_to_trash, args=(path,), size=1
)]
def touch(self, path):
raise self._operation_not_implemented()
def copy(self, src_url, dst_url):
+ """Unlike other methods, receives full URLs (with scheme), not paths."""
raise self._operation_not_implemented()
def prepare_copy(self, src_url, dst_url):
if self.copy.__func__ is FileSystem.copy:
@@ -161,6 +167,7 @@ def prepare_copy(self, src_url, dst_url):
fn=self.copy, args=(src_url, dst_url)
)]
def move(self, src_url, dst_url):
+ """Unlike other methods, receives full URLs (with scheme), not paths."""
raise self._operation_not_implemented()
def prepare_move(self, src_url, dst_url):
if self.move.__func__ is FileSystem.move:
diff --git a/src/main/python/fman/impl/fs_cache.py b/src/main/python/fman/impl/fs_cache.py
index ad665b6c..bd8b8b32 100644
--- a/src/main/python/fman/impl/fs_cache.py
+++ b/src/main/python/fman/impl/fs_cache.py
@@ -1,37 +1,89 @@
-from collections import defaultdict
-from threading import Lock
+# Lock ordering (acquire top-to-bottom, never invert):
+# Cache._lock
+# CacheItem._children_lock (per-node, parent before child)
+# CacheItem._attr_locks_lock (per-node)
+# CacheItem attr RLock (per-attr)
+# Cache._lock must NOT be held when calling compute_value in query(),
+# because compute_value may trigger nested cache operations.
+from threading import Lock, RLock
class Cache:
def __init__(self):
+ self._lock = Lock()
self._root = CacheItem()
+ self._generation = 0
def put(self, path, attr, value):
- self._root.update_child(path).put(attr, value)
+ with self._lock:
+ self._root.update_child(path).put(attr, value)
def get(self, path, attr):
- return self._root.get_child(path).get(attr)
+ with self._lock:
+ return self._root.get_child(path).get(attr)
def query(self, path, attr, compute_value):
- return self._root.update_child(path).query(attr, compute_value)
+ while True:
+ with self._lock:
+ gen = self._generation
+ item = self._root.update_child(path)
+ result = item.query(attr, compute_value)
+ with self._lock:
+ if self._generation == gen:
+ return result
+ item.clear_attr(attr)
+ def mutate(self, path, attr, fn):
+ with self._lock:
+ try:
+ item = self._root.get_child(path)
+ value = item.get(attr)
+ except KeyError:
+ return
+ fn(value)
def clear(self, path):
- if not path:
- self._root = CacheItem()
- else:
+ with self._lock:
+ self._generation += 1
+ if not path:
+ self._root = CacheItem()
+ else:
+ try:
+ self._root.delete_child(path)
+ except KeyError:
+ pass
+ def clear_attr(self, path, attr):
+ with self._lock:
try:
- self._root.delete_child(path)
+ item = self._root.get_child(path)
except KeyError:
- pass
+ return
+ item.clear_attr(attr)
class CacheItem:
def __init__(self):
self._children = {}
+ self._children_lock = Lock()
self._attrs = {}
- self._attr_locks = defaultdict(Lock)
+ self._attr_locks_lock = Lock()
+ self._attr_locks = {}
def put(self, attr, value):
- self._attrs[attr] = value
+ with self._attr_locks_lock:
+ if attr not in self._attr_locks:
+ self._attr_locks[attr] = RLock()
+ lock = self._attr_locks[attr]
+ with lock:
+ self._attrs[attr] = value
def get(self, attr):
return self._attrs[attr]
+ def clear_attr(self, attr):
+ with self._attr_locks_lock:
+ lock = self._attr_locks.pop(attr, None)
+ if lock:
+ with lock:
+ self._attrs.pop(attr, None)
+ else:
+ self._attrs.pop(attr, None)
def query(self, attr, compute_value):
- # Because `defaultdict` and `Lock` are implemented in C, they do not
- # release the GIL and the dict access is atomic:
- with self._attr_locks[attr]:
+ with self._attr_locks_lock:
+ if attr not in self._attr_locks:
+ self._attr_locks[attr] = RLock()
+ lock = self._attr_locks[attr]
+ with lock:
try:
return self._attrs[attr]
except KeyError:
@@ -57,4 +109,4 @@ def delete_child(self, path):
if len(parts) == 1:
del self._children[parts[0]]
else:
- self._children[parts[0]].delete_child(parts[1])
\ No newline at end of file
+ self._children[parts[0]].delete_child(parts[1])
diff --git a/src/main/python/fman/impl/model/__init__.py b/src/main/python/fman/impl/model/__init__.py
index 002a15e7..464ebb4a 100644
--- a/src/main/python/fman/impl/model/__init__.py
+++ b/src/main/python/fman/impl/model/__init__.py
@@ -20,6 +20,8 @@ class SortedFileSystemModel(QSortFilterProxyModel):
sort_order_changed = pyqtSignal(int, int)
transaction_ended = pyqtSignal()
+ _MAX_VISITED = 512
+
def __init__(self, parent, fs, null_location):
super().__init__(parent)
self._fs = fs
@@ -105,6 +107,9 @@ def _set_location_main(
)
self.setSourceModel(new_model)
self._connect_signals(new_model)
+ if len(self._already_visited) > self._MAX_VISITED:
+ self._already_visited.clear()
+ self._already_visited.add(url)
self._already_visited.add(url)
self.location_changed.emit(url)
order = Qt.AscendingOrder if ascending else Qt.DescendingOrder
@@ -146,21 +151,19 @@ def url(self, index):
def find(self, url):
return self.mapFromSource(self.sourceModel().find(url))
def _on_file_removed(self, url):
+ if not self.sourceModel():
+ return
if is_pardir(url, self.get_location()):
- dir_ = dirname(url)
- if dir_ == url:
- self.set_location(self._null_location)
- else:
+ while True:
+ dir_ = dirname(url)
+ if dir_ == url:
+ self.set_location(self._null_location)
+ return
try:
self.set_location(dir_)
+ return
except OSError:
- # In a perfect world, would like to only handle
- # FileNotFoundError here. But there can of course also be
- # other reasons. For example, when on a network share on
- # Windows, we may get a PermissionError trying to list a
- # parent directory we don't have access to. So catch all
- # OSErrors and in the worst case go to null://.
- self._on_file_removed(dir_)
+ url = dir_
def _connect_signals(self, model):
# Would prefer signal.connect(self.signal.emit) here. But PyQt doesn't
# support it. So we need Python wrappers "_emit_...":
@@ -189,5 +192,11 @@ def _emit_sort_order_changed(self, column, order):
self.sort_order_changed.emit(column, order)
def _emit_transaction_ended(self):
self.transaction_ended.emit()
+ def shutdown(self):
+ self._fs.file_removed.remove_callback(self._on_file_removed)
+ model = self.sourceModel()
+ if model:
+ self._disconnect_signals(model)
+ model.shutdown()
def __str__(self):
return '<%s: %s>' % (self.__class__.__name__, self.get_location())
\ No newline at end of file
diff --git a/src/main/python/fman/impl/model/diff.py b/src/main/python/fman/impl/model/diff.py
index aa97d82c..f420d95e 100644
--- a/src/main/python/fman/impl/model/diff.py
+++ b/src/main/python/fman/impl/model/diff.py
@@ -53,7 +53,8 @@ def _move_row(self, src, dest):
def _update_row(self, i, row):
self._result.append(DiffEntry(i, i + 1, i, [row]))
self._old_rows[i] = row
- assert self._key_fn(row) == self._old_keys[i]
+ if self._key_fn(row) != self._old_keys[i]:
+ raise RuntimeError('Key mismatch after update')
def join(diff_entries):
if not diff_entries:
@@ -135,8 +136,10 @@ def apply(self, insert, move, update, remove):
def _type(self):
if self._does_cut:
if self.rows:
- assert len(self.rows) == self.cut_end - self.cut_start
- assert self.cut_start == self.insert_start
+ if len(self.rows) != self.cut_end - self.cut_start:
+ raise RuntimeError('Row count mismatch in update entry')
+ if self.cut_start != self.insert_start:
+ raise RuntimeError('cut_start != insert_start in update entry')
return 'update'
else:
if self.insert_start != -1:
@@ -144,8 +147,10 @@ def _type(self):
else:
return 'remove'
else:
- assert self.rows
- assert self.insert_start != -1
+ if not self.rows:
+ raise RuntimeError('Insert entry has no rows')
+ if self.insert_start == -1:
+ raise RuntimeError('Insert entry has no insert_start')
return 'insert'
@property
def _does_cut(self):
diff --git a/src/main/python/fman/impl/model/icon_provider.py b/src/main/python/fman/impl/model/icon_provider.py
index 482a794c..e389e95d 100644
--- a/src/main/python/fman/impl/model/icon_provider.py
+++ b/src/main/python/fman/impl/model/icon_provider.py
@@ -5,6 +5,7 @@
from PyQt5.QtCore import QFileInfo
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import QFileIconProvider
+from threading import Lock
import logging
import sys
@@ -21,6 +22,7 @@ def __init__(self, qt_icon_provider, fs, cache_dir):
f.suffix: self._get_qt_icon(f)
for f in Path(cache_dir).glob('file*')
}
+ self._cache_lock = Lock()
def get_icon(self, url):
scheme, path = splitscheme(url)
if scheme == 'file://':
@@ -32,14 +34,13 @@ def get_icon(self, url):
if self._fs.is_dir(url):
return self._folder_icon
suffix = PurePosixPath(path).suffix
- if suffix not in self._cache:
- surrogate = Path(self._cache_dir, 'file' + suffix)
- with surrogate.open('w') as f:
- # At least Gnome doesn't display a proper icon unless the file
- # has some contents. So give it some:
- f.write('fman')
- self._cache[suffix] = self._get_qt_icon(surrogate)
- return self._cache[suffix]
+ with self._cache_lock:
+ if suffix not in self._cache:
+ surrogate = Path(self._cache_dir, 'file' + suffix)
+ with surrogate.open('w') as f:
+ f.write('fman')
+ self._cache[suffix] = self._get_qt_icon(surrogate)
+ return self._cache[suffix]
def _get_qt_icon(self, path):
if not isinstance(path, str):
path = str(path)
@@ -105,7 +106,7 @@ def _query_gio_info(self, file_path, *args):
try:
return gio_file.query_info(*args)
except self.GLib.GError as e:
- if e.message and e.message.endswith('No such file or directory'):
+ if str(e).endswith('No such file or directory'):
raise filenotfounderror(file_path)
else:
raise
diff --git a/src/main/python/fman/impl/model/record_files.py b/src/main/python/fman/impl/model/record_files.py
index a4d92d15..8e0117c4 100644
--- a/src/main/python/fman/impl/model/record_files.py
+++ b/src/main/python/fman/impl/model/record_files.py
@@ -92,6 +92,7 @@ def _preprocess_existing(self):
else:
if not self._m_accepts(file_):
to_remove.append(rownum)
+ self._m_files[file_.url] = file_
continue
if file_ != old_file:
to_update.append((rownum, file_))
diff --git a/src/main/python/fman/impl/model/table.py b/src/main/python/fman/impl/model/table.py
index 33fe1c80..c1d17ff6 100644
--- a/src/main/python/fman/impl/model/table.py
+++ b/src/main/python/fman/impl/model/table.py
@@ -151,7 +151,7 @@ def insert(self, rows, first_rownum):
new_keys = {row.key: first_rownum + i for i, row in enumerate(rows)}
with self._lock:
# Perform this check here, once we have the lock:
- if first_rownum < 0 or first_rownum > len(self._rows) + 1:
+ if first_rownum < 0 or first_rownum > len(self._rows):
raise ValueError('Invalid first_rownum: %d' % first_rownum)
num_rows = len(rows)
for row in self._rows[first_rownum:]:
diff --git a/src/main/python/fman/impl/model/worker.py b/src/main/python/fman/impl/model/worker.py
index 51066233..df74de27 100644
--- a/src/main/python/fman/impl/model/worker.py
+++ b/src/main/python/fman/impl/model/worker.py
@@ -21,7 +21,7 @@ def submit(self, priority, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown:
return
- self._queue.put(WorkItem(priority, fn, *args, *kwargs))
+ self._queue.put(WorkItem(priority, fn, *args, **kwargs))
def shutdown(self):
with self._shutdown_lock:
self._shutdown = True
@@ -56,7 +56,7 @@ def __lt__(self, other):
return NotImplemented
def __eq__(self, other):
try:
- return self._fn, self._args, self._kwargs, self._priority == \
- other._fn, other._args, other._kwargs, other._priority
+ return (self._fn, self._args, self._kwargs, self._priority) == \
+ (other._fn, other._args, other._kwargs, other._priority)
except AttributeError:
return NotImplemented
\ No newline at end of file
diff --git a/src/main/python/fman/impl/plugins/command_registry.py b/src/main/python/fman/impl/plugins/command_registry.py
index e7204763..7bb1a0e6 100644
--- a/src/main/python/fman/impl/plugins/command_registry.py
+++ b/src/main/python/fman/impl/plugins/command_registry.py
@@ -153,9 +153,11 @@ def _set_context(self, pane, file_under_cursor=_DEFAULT):
if file_under_cursor is not self._DEFAULT:
cm = pane._override_file_under_cursor(file_under_cursor)
cm.__enter__()
- yield
- if file_under_cursor is not self._DEFAULT:
- cm.__exit__(None, None, None)
+ try:
+ yield
+ finally:
+ if file_under_cursor is not self._DEFAULT:
+ cm.__exit__(None, None, None)
def _get_default_aliases(cmd_class):
return re.sub(r'([a-z])([A-Z])', r'\1 \2', cmd_class.__name__)\
diff --git a/src/main/python/fman/impl/plugins/error.py b/src/main/python/fman/impl/plugins/error.py
index 8b19d965..c6797555 100644
--- a/src/main/python/fman/impl/plugins/error.py
+++ b/src/main/python/fman/impl/plugins/error.py
@@ -46,8 +46,8 @@ def handle_system_exit(self, code=0):
self._app.exit(code)
def on_main_window_shown(self, main_window):
self._main_window = main_window
- if self._pending_error_messages:
- self._main_window.show_alert(self._pending_error_messages[0])
+ for message in self._pending_error_messages:
+ self._main_window.show_alert(message)
def _get_plugin_traceback(self, exc):
if isinstance(exc, ThemeError):
return exc.description
diff --git a/src/main/python/fman/impl/plugins/mother_fs.py b/src/main/python/fman/impl/plugins/mother_fs.py
index 552b4f59..2f975abb 100644
--- a/src/main/python/fman/impl/plugins/mother_fs.py
+++ b/src/main/python/fman/impl/plugins/mother_fs.py
@@ -51,9 +51,9 @@ def iterdir(self, url):
child, path = self._split(url)
def compute_value():
iterator = getattr(child, 'iterdir')(path)
- if hasattr(iterator, '__next__'):
- iterator = CachedIterator(iterator)
- return iterator
+ if not hasattr(iterator, '__next__'):
+ iterator = iter(iterator)
+ return CachedIterator(iterator)
return child.cache.query(path, 'iterdir', compute_value)
def query(self, url, fs_method_name):
child, path = self._split(url)
@@ -170,6 +170,7 @@ def clear_cache(self, url):
child.cache.clear(path)
def notify_file_added(self, url):
child, path = self._split(url)
+ child.cache.clear(path)
child.notify_file_added(path)
def notify_file_changed(self, url):
child, path = self._split(url)
@@ -179,9 +180,11 @@ def notify_file_removed(self, url):
child.notify_file_removed(path)
def _on_file_added(self, url):
self._add_to_parent(url)
+ self._clear_parent_stat(url)
self.file_added.trigger(url)
def _on_file_removed(self, url):
self._remove(url)
+ self._clear_parent_stat(url)
self.file_removed.trigger(url)
def _split(self, url):
scheme, path = splitscheme(url)
@@ -194,58 +197,84 @@ def _remove(self, url):
child, path = self._split(url)
child.cache.clear(path)
parent_path = splitscheme(dirname(url))[1]
+ name = basename(url)
+ child.cache.mutate(parent_path, 'iterdir', lambda ci: ci.remove(name))
+ def _clear_parent_stat(self, url):
+ parent = dirname(url)
try:
- parent_files = child.cache.get(parent_path, 'iterdir')
- except KeyError:
- pass
- else:
- try:
- parent_files.remove(basename(url))
- except ValueError:
- pass
+ child, parent_path = self._split(parent)
+ except FileNotFoundError:
+ return
+ for attr in ('stat', 'icon'):
+ child.cache.clear_attr(parent_path, attr)
def _add_to_parent(self, url):
parent = dirname(url)
- child, parent_path = self._split(parent)
try:
- parent_files = child.cache.get(parent_path, 'iterdir')
- except KeyError:
- pass
- else:
- parent_files.append(basename(url))
+ child, parent_path = self._split(parent)
+ except FileNotFoundError:
+ return
+ name = basename(url)
+ child.cache.mutate(parent_path, 'iterdir', lambda ci: ci.append(name))
+# Lock ordering within CachedIterator:
+# _lock (protects _items list and _item_counts dict)
+# _source_lock (serializes source iterator advancement)
+# _lock is never held when acquiring _source_lock. Inside _source_lock,
+# _lock is briefly re-acquired for snapshots — this is safe because no
+# code path holds _lock and then acquires _source_lock.
class CachedIterator:
def __init__(self, source):
self._source = source
self._lock = Lock()
+ self._source_lock = Lock()
self._items = []
self._item_counts = {}
def remove(self, item):
with self._lock:
self._record(item, delta=-1)
+ self._item_counts[item] = min(self._item_counts[item], 0)
def append(self, item):
# N.B.: Behaves like set#add(...), not like list#append(...)!
with self._lock:
self._record(item)
+ if self._item_counts[item] <= 0:
+ self._item_counts[item] = 1
+ if item not in self._items:
+ self._items.append(item)
def __iter__(self):
return _CachedIterator(self)
def get_next(self, pointer):
+ original_pointer = pointer
with self._lock:
- for pointer in range(pointer, len(self._items)):
- item = self._items[pointer]
+ for i in range(pointer, len(self._items)):
+ item = self._items[i]
if self._item_counts[item] > 0:
- return pointer + 1, item
+ return i + 1, item
+ pointer = len(self._items)
+ with self._source_lock:
while True:
+ with self._lock:
+ for p in range(original_pointer, len(self._items)):
+ item = self._items[p]
+ if self._item_counts[item] > 0:
+ return p + 1, item
+ pointer = len(self._items)
value = next(self._source) # Eventually raises StopIteration
- if self._record(value):
- return len(self._items), value
+ with self._lock:
+ if self._record(value):
+ return len(self._items), value
def _record(self, value, delta=1):
try:
- self._item_counts[value] += delta
- return False
+ old_count = self._item_counts[value]
+ self._item_counts[value] = old_count + delta
+ return old_count <= 0 and old_count + delta > 0
except KeyError:
+ if delta < 0:
+ self._item_counts[value] = delta
+ return False
self._items.append(value)
self._item_counts[value] = delta
- return True
+ return delta > 0
class _CachedIterator:
def __init__(self, parent):
@@ -255,4 +284,4 @@ def __iter__(self):
return self
def __next__(self):
self._pointer, result = self._parent.get_next(self._pointer)
- return result
\ No newline at end of file
+ return result
diff --git a/src/main/python/fman/impl/session.py b/src/main/python/fman/impl/session.py
index 58278245..18144e70 100644
--- a/src/main/python/fman/impl/session.py
+++ b/src/main/python/fman/impl/session.py
@@ -68,13 +68,6 @@ def _show_startup_messages(self, main_window):
'Updated to v%s. ' \
'Changelog' % self._fman_version
main_window.show_status_message(status_message, timeout_secs=5)
- def _get_startup_message(self):
- previous_version = self._settings.get('fman_version', None)
- if not previous_version or previous_version == self._fman_version:
- return 'v%s ready.' % self._fman_version
- return 'Updated to v%s. ' \
- 'Changelog' \
- % self._fman_version
def _init_panes(self, panes, pane_infos, paths_on_cmdline):
with ThreadPoolExecutor(max_workers=len(panes)) as executor:
futures = [
diff --git a/src/main/python/fman/impl/util/__init__.py b/src/main/python/fman/impl/util/__init__.py
index 519176cc..64a3fa37 100644
--- a/src/main/python/fman/impl/util/__init__.py
+++ b/src/main/python/fman/impl/util/__init__.py
@@ -2,6 +2,7 @@
from os import listdir, strerror
from os.path import join, basename, expanduser, dirname, realpath, relpath, \
pardir, splitdrive
+from threading import Lock
import errno
import os
@@ -63,12 +64,17 @@ def __init__(self, *args):
class Event:
def __init__(self):
self._callbacks = []
+ self._lock = Lock()
def add_callback(self, callback):
- self._callbacks.append(callback)
+ with self._lock:
+ self._callbacks.append(callback)
def remove_callback(self, callback):
- self._callbacks.remove(callback)
+ with self._lock:
+ self._callbacks.remove(callback)
def trigger(self, *args):
- for callback in self._callbacks:
+ with self._lock:
+ callbacks = list(self._callbacks)
+ for callback in callbacks:
callback(*args)
# Copied from core.util:
diff --git a/src/main/python/fman/impl/util/path.py b/src/main/python/fman/impl/util/path.py
index b5ffff8a..7105975f 100644
--- a/src/main/python/fman/impl/util/path.py
+++ b/src/main/python/fman/impl/util/path.py
@@ -34,5 +34,8 @@ def normalize(path_):
if path_ == '.':
path_ = ''
# Resolve a/../b
- path_ = re.subn(r'(^|/)([^/]+)/\.\.(?:$|/)', r'\1', path_)[0]
+ while True:
+ path_, count = re.subn(r'(^|/)([^/]+)/\.\.(?:$|/)', r'\1', path_)
+ if not count:
+ break
return path_.rstrip('/')
\ No newline at end of file
diff --git a/src/main/python/fman/impl/util/qt/__init__.py b/src/main/python/fman/impl/util/qt/__init__.py
index 9d77ee96..8baddbad 100644
--- a/src/main/python/fman/impl/util/qt/__init__.py
+++ b/src/main/python/fman/impl/util/qt/__init__.py
@@ -16,8 +16,9 @@ def disable_window_animations_mac(window):
# penalties and leads to subtle changes in behaviour. We therefore wait for
# the Show event:
def eventFilter(target, event):
+ from ctypes import c_void_p
from objc import objc_object
- view = objc_object(c_void_p=int(target.winId()))
+ view = objc_object(c_void_p=c_void_p(int(target.winId())))
NSWindowAnimationBehaviorNone = 2
view.window().setAnimationBehavior_(NSWindowAnimationBehaviorNone)
FilterEventOnce(window, QEvent.Show, eventFilter)
diff --git a/src/main/python/fman/impl/util/qt/thread.py b/src/main/python/fman/impl/util/qt/thread.py
index 64a282dd..f51161f2 100644
--- a/src/main/python/fman/impl/util/qt/thread.py
+++ b/src/main/python/fman/impl/util/qt/thread.py
@@ -54,23 +54,23 @@ def instance(cls):
return cls._INSTANCE
def __init__(self, app):
self._pending_tasks = []
+ self._tasks_lock = Lock()
self._app_is_about_to_quit = False
app.aboutToQuit.connect(self._about_to_quit)
def _about_to_quit(self):
- self._app_is_about_to_quit = True
- for task in self._pending_tasks:
- task.set_exception(SystemExit())
- task.has_run.set()
+ with self._tasks_lock:
+ self._app_is_about_to_quit = True
+ for task in self._pending_tasks:
+ task.set_exception(SystemExit())
+ task.has_run.set()
def run_in_thread(self, thread, f, args, kwargs):
if QThread.currentThread() == thread:
return f(*args, **kwargs)
- elif self._app_is_about_to_quit:
- # In this case, the target thread's event loop most likely is not
- # running any more. This would mean that our task (which is
- # submitted to the event loop via signals/slots) is never run.
- raise SystemExit()
- task = Task(f, args, kwargs)
- self._pending_tasks.append(task)
+ with self._tasks_lock:
+ if self._app_is_about_to_quit:
+ raise SystemExit()
+ task = Task(f, args, kwargs)
+ self._pending_tasks.append(task)
try:
receiver = Receiver(task)
receiver.moveToThread(thread)
@@ -80,7 +80,8 @@ def run_in_thread(self, thread, f, args, kwargs):
task.has_run.wait()
return task.result
finally:
- self._pending_tasks.remove(task)
+ with self._tasks_lock:
+ self._pending_tasks.remove(task)
class Task:
def __init__(self, fn, args, kwargs):
@@ -92,7 +93,8 @@ def __init__(self, fn, args, kwargs):
def __call__(self):
try:
self._result = self._fn(*self._args, **self._kwargs)
- except Exception as e:
+ self._exception = None
+ except BaseException as e:
self._exception = e
finally:
self.has_run.set()
diff --git a/src/main/python/fman/impl/widgets.py b/src/main/python/fman/impl/widgets.py
index c92c08fc..5a591d98 100644
--- a/src/main/python/fman/impl/widgets.py
+++ b/src/main/python/fman/impl/widgets.py
@@ -37,7 +37,7 @@ def exit(self, returnCode=0):
def set_style_sheet(self, stylesheet):
self.setStyleSheet(stylesheet)
def _on_state_changed(self, new_state):
- if new_state == Qt.ApplicationActive:
+ if new_state == Qt.ApplicationActive and self._main_window is not None:
for pane in self._main_window.get_panes():
pane.reload()
@@ -155,7 +155,7 @@ def get_sort_column(self):
return column, ascending
@run_in_main_thread
def get_column_widths(self):
- return [self._file_view.columnWidth(i) for i in (0, 1)]
+ return [self._file_view.columnWidth(i) for i in range(self._model.columnCount())]
@run_in_main_thread
def set_column_widths(self, column_widths):
num_columns = self._model.columnCount()
diff --git a/src/main/resources/base/Plugins/Core/core/commands/__init__.py b/src/main/resources/base/Plugins/Core/core/commands/__init__.py
index 319553d4..b6c88f53 100644
--- a/src/main/resources/base/Plugins/Core/core/commands/__init__.py
+++ b/src/main/resources/base/Plugins/Core/core/commands/__init__.py
@@ -17,7 +17,7 @@
from io import UnsupportedOperation
from itertools import chain
from os import strerror
-from os.path import basename, pardir
+from os.path import pardir
from pathlib import PurePath
from PyQt5.QtCore import QUrl
from PyQt5.QtGui import QDesktopServices
@@ -1730,7 +1730,7 @@ def on_command(self, command_name, args):
except (KeyError, ValueError):
return None
if scheme == 'file://':
- new_scheme = _get_handler_for_archive(basename(path))
+ new_scheme = _get_handler_for_archive(basename(url))
if new_scheme:
try:
if is_dir(url):
diff --git a/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py b/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py
index 837f2bc2..a68a38d6 100644
--- a/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py
+++ b/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py
@@ -351,7 +351,7 @@ def _read_directory(self, dir_path):
child_contents = self._read_directory(child)
else:
child_contents = child.read_text()
- result[child.name] = child_contents
+ result[normalize('NFC', child.name)] = child_contents
return result
def _expect_zip_contents(self, contents, zip_file_path):
with TemporaryDirectory() as tmp_dir:
diff --git a/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py b/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py
index 2d5d170a..927613c9 100644
--- a/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py
+++ b/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py
@@ -3,7 +3,7 @@
from fman.fs import FileSystem, cached
from fman.impl.plugins.mother_fs import MotherFileSystem, CachedIterator
from fman_unittest.impl.model import StubFileSystem
-from threading import Thread, Lock, Event
+from threading import Thread, Barrier, Lock, Event
from time import sleep
from unittest import TestCase
@@ -141,6 +141,51 @@ def test_relative_paths(self):
self.assertTrue(mother_fs.is_dir('stub://dir'))
mother_fs.move('stub://a/b', 'stub://a/../b')
self.assertTrue(mother_fs.exists('stub://b'))
+ def test_file_added_clears_parent_stat(self):
+ fs = StubFileSystem({
+ 'a': {'is_dir': True, 'files': []}
+ })
+ mother_fs = self._create_mother_fs(fs)
+ mother_fs.is_dir('stub://a')
+ self.assertIsNotNone(fs.cache.get('a', 'is_dir'))
+ mother_fs.touch('stub://a/b')
+ with self.assertRaises(KeyError):
+ fs.cache.get('a', 'stat')
+ def test_file_removed_clears_parent_stat(self):
+ fs = StubFileSystem({
+ 'a': {'is_dir': True, 'files': ['b']},
+ 'a/b': {}
+ })
+ mother_fs = self._create_mother_fs(fs)
+ mother_fs.is_dir('stub://a')
+ mother_fs.delete('stub://a/b')
+ with self.assertRaises(KeyError):
+ fs.cache.get('a', 'stat')
+ def test_iterdir_returns_cached_iterator(self):
+ fs = StubFileSystem({
+ 'a': {'is_dir': True, 'files': ['b', 'c']}
+ })
+ mother_fs = self._create_mother_fs(fs)
+ result = mother_fs.iterdir('stub://a')
+ self.assertIsInstance(result, CachedIterator)
+ def test_iterdir_cached_is_same_instance(self):
+ fs = StubFileSystem({
+ 'a': {'is_dir': True, 'files': ['b']}
+ })
+ mother_fs = self._create_mother_fs(fs)
+ first = mother_fs.iterdir('stub://a')
+ second = mother_fs.iterdir('stub://a')
+ self.assertIs(first, second)
+ def test_iterdir_list_source_becomes_cached_iterator(self):
+ """Even when underlying FS returns plain list, iterdir wraps it."""
+ fs = StubFileSystem({
+ 'a': {'is_dir': True, 'files': ['x', 'y']}
+ })
+ mother_fs = self._create_mother_fs(fs)
+ result = mother_fs.iterdir('stub://a')
+ self.assertIsInstance(result, CachedIterator)
+ self.assertEqual(Counter(['x', 'y']), Counter(list(result)))
+ self.assertEqual(Counter(['x', 'y']), Counter(list(result)))
def _create_mother_fs(self, fs):
result = MotherFileSystem(None)
result.add_child(fs.scheme, fs)
@@ -264,6 +309,138 @@ def test_add_remove_out_of_order(self):
iterable.remove(1)
with self.assertRaises(StopIteration):
next(iterator)
+ def test_remove_then_append_same_item(self):
+ iterable = CachedIterator(self._generate(1, 2))
+ iterator = iter(iterable)
+ self.assertEqual(1, next(iterator))
+ iterable.remove(2)
+ iterable.append(2)
+ self.assertEqual(2, next(iterator))
+ with self.assertRaises(StopIteration):
+ next(iterator)
+ self.assertEqual([1, 2], list(iterable))
+ def test_remove_before_yield_then_append(self):
+ iterable = CachedIterator(self._generate(1, 2, 3))
+ iterable.remove(3)
+ iterable.append(3)
+ result = list(iterable)
+ self.assertIn(3, result)
+ self.assertEqual(Counter([1, 2, 3]), Counter(result))
+ def test_pre_remove_unknown_item_then_source_yields_it(self):
+ iterable = CachedIterator(self._generate(1, 2))
+ iterable.remove(2)
+ result = list(iterable)
+ self.assertEqual([1], result)
+ def test_concurrent_remove_append(self):
+ iterable = CachedIterator(self._generate(*range(100)))
+ results_before = []
+ results_after = []
+ barrier = Event()
+ def reader():
+ barrier.wait()
+ results_after.extend(list(iterable))
+ for i in range(50, 100):
+ iterable.remove(i)
+ for i in range(50, 100):
+ iterable.append(i)
+ results_before = list(iterable)
+ self.assertEqual(Counter(range(100)), Counter(results_before))
+ def test_get_next_pointer_after_all_removed(self):
+ """get_next must advance pointer past cached items when all are dead."""
+ iterable = CachedIterator(self._generate(1, 2, 3, 4))
+ iterator = iter(iterable)
+ self.assertEqual(1, next(iterator))
+ self.assertEqual(2, next(iterator))
+ iterable.remove(3)
+ iterable.remove(4)
+ iterable.append(5)
+ result = list(iterator)
+ self.assertIn(5, result)
+ self.assertNotIn(3, result)
+ self.assertNotIn(4, result)
+ def test_concurrent_iterate_and_mutate(self):
+ """Multiple readers + one mutator must not crash or produce dupes."""
+ items = list(range(50))
+ iterable = CachedIterator(self._generate(*items))
+ errors = []
+ results = [None, None]
+ barrier = Barrier(3)
+ def reader(idx):
+ barrier.wait()
+ try:
+ results[idx] = list(iterable)
+ except Exception as e:
+ errors.append(e)
+ def mutator():
+ barrier.wait()
+ for i in range(50, 80):
+ iterable.append(i)
+ for i in range(10):
+ iterable.remove(i)
+ threads = [
+ Thread(target=reader, args=(0,)),
+ Thread(target=reader, args=(1,)),
+ Thread(target=mutator)
+ ]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ self.assertEqual([], errors)
+ for result in results:
+ if result is not None:
+ self.assertEqual(len(result), len(set(result)),
+ 'Duplicates found: %r' % result)
+ def test_concurrent_remove_append_stress(self):
+ """Hammer remove+append from many threads, no crash or lost items."""
+ iterable = CachedIterator(self._generate(*range(200)))
+ list(iterable)
+ errors = []
+ barrier = Barrier(4)
+ def remove_thread(start, end):
+ barrier.wait()
+ for i in range(start, end):
+ try:
+ iterable.remove(i)
+ except Exception as e:
+ errors.append(e)
+ def append_thread(start, end):
+ barrier.wait()
+ for i in range(start, end):
+ try:
+ iterable.append(i)
+ except Exception as e:
+ errors.append(e)
+ threads = [
+ Thread(target=remove_thread, args=(0, 100)),
+ Thread(target=remove_thread, args=(100, 200)),
+ Thread(target=append_thread, args=(0, 100)),
+ Thread(target=append_thread, args=(100, 200))
+ ]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ self.assertEqual([], errors)
+ result = list(iterable)
+ self.assertEqual(len(result), len(set(result)))
+ def test_concurrent_iteration_with_slow_source(self):
+ """Two iterators consuming a slow source must not lose items."""
+ items = list(range(20))
+ iterable = CachedIterator(self._generate_slowly(*items))
+ results = [None, None]
+ barrier = Barrier(2)
+ def reader(idx):
+ barrier.wait()
+ results[idx] = list(iterable)
+ t1 = Thread(target=reader, args=(0,))
+ t2 = Thread(target=reader, args=(1,))
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ for result in results:
+ self.assertEqual(sorted(items), sorted(result))
def _generate(self, *args):
yield from args
def _generate_slowly(self, *args):
@@ -272,4 +449,4 @@ def _generate_slowly(self, *args):
yield arg
def _consume(self, started, iterable):
started.set()
- return list(iterable)
\ No newline at end of file
+ return list(iterable)
diff --git a/src/unittest/python/fman_unittest/impl/test_fs.py b/src/unittest/python/fman_unittest/impl/test_fs.py
new file mode 100644
index 00000000..88921dcc
--- /dev/null
+++ b/src/unittest/python/fman_unittest/impl/test_fs.py
@@ -0,0 +1,47 @@
+from fman.fs import FileSystem
+from threading import Thread, Barrier, Lock
+from unittest import TestCase
+
+class NotifyFileChangedTest(TestCase):
+ def test_callback_called(self):
+ fs = FileSystem()
+ results = []
+ fs._file_changed_callbacks['test'] = [lambda url: results.append(url)]
+ fs.notify_file_changed('test')
+ self.assertEqual([fs.scheme + 'test'], results)
+ def test_no_callbacks_for_path(self):
+ fs = FileSystem()
+ fs.notify_file_changed('nonexistent')
+ def test_callback_removal_during_notify(self):
+ fs = FileSystem()
+ results = []
+ callbacks = []
+ def remove_self(url):
+ with fs._file_changed_callbacks_lock:
+ callbacks.remove(remove_self)
+ results.append('removed')
+ def second(url):
+ results.append('second')
+ callbacks.extend([remove_self, second])
+ fs._file_changed_callbacks['test'] = callbacks
+ fs.notify_file_changed('test')
+ self.assertEqual(['removed', 'second'], results)
+ def test_thread_safety(self):
+ fs = FileSystem()
+ call_count = 0
+ count_lock = Lock()
+ def counter(url):
+ nonlocal call_count
+ with count_lock:
+ call_count += 1
+ fs._file_changed_callbacks['test'] = [counter]
+ barrier = Barrier(10)
+ def notify():
+ barrier.wait()
+ fs.notify_file_changed('test')
+ threads = [Thread(target=notify) for _ in range(10)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ self.assertEqual(10, call_count)
diff --git a/src/unittest/python/fman_unittest/impl/test_fs_cache.py b/src/unittest/python/fman_unittest/impl/test_fs_cache.py
index 55564880..8da805d7 100644
--- a/src/unittest/python/fman_unittest/impl/test_fs_cache.py
+++ b/src/unittest/python/fman_unittest/impl/test_fs_cache.py
@@ -1,5 +1,6 @@
-from fman.impl.fs_cache import Cache
-from threading import Thread
+from fman.impl.fs_cache import Cache, CacheItem
+from collections import defaultdict
+from threading import Thread, Barrier, Event, RLock
from time import sleep
from unittest import TestCase
@@ -46,6 +47,202 @@ def compute_value():
thread_1.join()
thread_2.join()
self.assertEqual(1, len(calls))
+ def test_query_reentrant_rlock(self):
+ outer_called = []
+ def compute_outer():
+ outer_called.append(1)
+ self.cache.query('nested', 'attr', lambda: 'inner')
+ return 'outer'
+ result = self.cache.query('top', 'val', compute_outer)
+ self.assertEqual('outer', result)
+ self.assertEqual('inner', self.cache.get('nested', 'attr'))
+ def test_concurrent_put_and_clear(self):
+ errors = []
+ barrier = Barrier(2)
+ def writer():
+ barrier.wait()
+ for i in range(100):
+ try:
+ self.cache.put('path/%d' % i, 'attr', i)
+ except Exception as e:
+ errors.append(e)
+ def clearer():
+ barrier.wait()
+ for _ in range(100):
+ try:
+ self.cache.clear('')
+ except Exception as e:
+ errors.append(e)
+ t1 = Thread(target=writer)
+ t2 = Thread(target=clearer)
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ self.assertEqual([], errors)
+ def test_concurrent_query_different_paths(self):
+ results = {}
+ barrier = Barrier(2)
+ def query_path(path, value):
+ barrier.wait()
+ results[path] = self.cache.query(path, 'attr', lambda: value)
+ t1 = Thread(target=query_path, args=('a', 1))
+ t2 = Thread(target=query_path, args=('b', 2))
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ self.assertEqual(1, results['a'])
+ self.assertEqual(2, results['b'])
+ def test_clear_attr(self):
+ self.cache.put('path', 'keep', 'yes')
+ self.cache.put('path', 'remove', 'no')
+ self.cache.clear_attr('path', 'remove')
+ self.assertEqual('yes', self.cache.get('path', 'keep'))
+ with self.assertRaises(KeyError):
+ self.cache.get('path', 'remove')
+ def test_clear_attr_nonexistent_path(self):
+ self.cache.clear_attr('nonexistent', 'attr')
+ def test_clear_attr_nonexistent_attr(self):
+ self.cache.put('path', 'exists', 1)
+ self.cache.clear_attr('path', 'nope')
+ self.assertEqual(1, self.cache.get('path', 'exists'))
+ def test_clear_attr_allows_recompute(self):
+ self.cache.query('p', 'a', lambda: 'first')
+ self.assertEqual('first', self.cache.get('p', 'a'))
+ self.cache.clear_attr('p', 'a')
+ self.cache.query('p', 'a', lambda: 'second')
+ self.assertEqual('second', self.cache.get('p', 'a'))
+ def test_clear_nonexistent_path(self):
+ self.cache.clear('nonexistent/path')
+ def test_nested_path_put_get(self):
+ self.cache.put('a/b/c', 'val', 42)
+ self.assertEqual(42, self.cache.get('a/b/c', 'val'))
+ def test_clear_parent_clears_children(self):
+ self.cache.put('a/b', 'val', 1)
+ self.cache.clear('a')
+ with self.assertRaises(KeyError):
+ self.cache.get('a/b', 'val')
+ def test_query_retries_on_concurrent_clear(self):
+ """Generation counter forces retry when clear() races with query."""
+ compute_count = [0]
+ compute_started = Event()
+ clear_done = Event()
+ def compute_value():
+ compute_count[0] += 1
+ if compute_count[0] == 1:
+ compute_started.set()
+ clear_done.wait(timeout=2)
+ return 'val_%d' % compute_count[0]
+ def do_clear():
+ compute_started.wait(timeout=2)
+ self.cache.clear('')
+ clear_done.set()
+ t = Thread(target=do_clear)
+ t.start()
+ result = self.cache.query('p', 'a', compute_value)
+ t.join()
+ self.assertEqual(2, compute_count[0])
+ self.assertEqual('val_2', result)
+ def test_mutate_safe_during_concurrent_clear(self):
+ """mutate must not crash when clear removes the item concurrently."""
+ errors = []
+ barrier = Barrier(2)
+ def mutator():
+ barrier.wait()
+ for _ in range(100):
+ self.cache.mutate('p', 'list', lambda v: v.append(0))
+ def clearer():
+ barrier.wait()
+ for _ in range(100):
+ self.cache.clear('p')
+ self.cache.put('p', 'list', [1, 2, 3])
+ self.cache.put('p', 'list', [1, 2, 3])
+ t1 = Thread(target=mutator)
+ t2 = Thread(target=clearer)
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ self.assertEqual([], errors)
+ def test_concurrent_clear_attr_and_query(self):
+ """clear_attr and query on same attr must not corrupt state."""
+ errors = []
+ barrier = Barrier(2)
+ def querier():
+ barrier.wait()
+ for i in range(100):
+ try:
+ self.cache.query('p', 'a', lambda: i)
+ except Exception as e:
+ errors.append(e)
+ def clearer():
+ barrier.wait()
+ for _ in range(100):
+ self.cache.clear_attr('p', 'a')
+ self.cache.put('p', 'a', 'initial')
+ t1 = Thread(target=querier)
+ t2 = Thread(target=clearer)
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ self.assertEqual([], errors)
+ def test_concurrent_query_clear_query_returns_fresh(self):
+ """After clear, next query recomputes — never returns stale data from
+ a generation that was already invalidated."""
+ for _ in range(20):
+ cache = Cache()
+ stale = [False]
+ compute_started = Event()
+ clear_done = Event()
+ def first_compute():
+ compute_started.set()
+ clear_done.wait(timeout=2)
+ return 'stale'
+ def second_compute():
+ return 'fresh'
+ def do_clear():
+ compute_started.wait(timeout=2)
+ cache.clear('')
+ clear_done.set()
+ t = Thread(target=do_clear)
+ t.start()
+ count = [0]
+ def compute():
+ count[0] += 1
+ if count[0] == 1:
+ return first_compute()
+ return second_compute()
+ result = cache.query('p', 'a', compute)
+ t.join()
+ self.assertEqual('fresh', result)
def setUp(self):
super().setUp()
- self.cache = Cache()
\ No newline at end of file
+ self.cache = Cache()
+
+class CacheItemTest(TestCase):
+ def test_attr_locks_are_reentrant(self):
+ item = CacheItem()
+ item.query('test', lambda: 'val')
+ lock = item._attr_locks['test']
+ lock.acquire()
+ acquired = lock.acquire(blocking=False)
+ self.assertTrue(acquired)
+ lock.release()
+ lock.release()
+ def test_update_child_creates_path(self):
+ item = CacheItem()
+ child = item.update_child('a/b/c')
+ child.put('key', 'val')
+ self.assertEqual('val', item.get_child('a/b/c').get('key'))
+ def test_delete_child_nested(self):
+ item = CacheItem()
+ item.update_child('a/b').put('k', 'v')
+ item.delete_child('a/b')
+ with self.assertRaises(KeyError):
+ item.get_child('a/b')
+ def test_get_child_nonexistent(self):
+ item = CacheItem()
+ with self.assertRaises(KeyError):
+ item.get_child('nope')