From b459b95950c8557a8f3d3f2d67f0a4420759f503 Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sat, 2 May 2026 03:28:01 +0200 Subject: [PATCH 1/7] - The comma-separated return creates a 7-element tuple instead of comparing two tuples, every equality check returns True - master -> main branch fix - prepare_trash now calls self.move_to_trash instead of self.delete, so plugin subclasses won't permanently delete files when the user expects trashing - NotImplementedError - unrecognized platforms fail fast with a clear message instead of a cryptic NameError - removed shadowing basename import - get_column_widths uses range so plugin-added columns get their widths saved/restored --- build.py | 8 ++++---- src/main/python/fman/__init__.py | 2 ++ src/main/python/fman/fs.py | 2 +- src/main/python/fman/impl/model/worker.py | 6 +++--- src/main/python/fman/impl/widgets.py | 2 +- .../resources/base/Plugins/Core/core/commands/__init__.py | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/build.py b/build.py index 60255c1a..f9988e72 100644 --- a/build.py +++ b/build.py @@ -96,7 +96,7 @@ def release(): settings_path, next_version + snapshot_suffix, 'Bump version for next development iteration' ) - git('push', '-u', 'origin', 'master') + git('push', '-u', 'origin', 'main') try: git('push', 'origin', release_tag) try: @@ -106,7 +106,7 @@ def release(): ' git pull\n' ' git checkout %s\n' ' python build.py release\n' - ' git checkout master\n\n' + ' git checkout main\n\n' 'on the other OSs now, then come back here and do:' '\n\n' ' python build.py post_release\n' @@ -117,7 +117,7 @@ def release(): raise except: git('revert', '--no-edit', revision_before + '..HEAD' ) - git('push', '-u', 'origin', 'master') + git('push', '-u', 'origin', 'main') revision_before = git('rev-parse', 'HEAD').rstrip() raise except: @@ -149,7 +149,7 @@ def post_release(): create_cloudfront_invalidation(cloudfront_items_to_invalidate) record_release_on_server() upload_core_to_github() - git('checkout', 'master') + git('checkout', 'main') def _prompt_for_next_version(release_version): next_version = _get_suggested_next_version(release_version) diff --git a/src/main/python/fman/__init__.py b/src/main/python/fman/__init__.py index ab1f58d1..eefe7c12 100644 --- a/src/main/python/fman/__init__.py +++ b/src/main/python/fman/__init__.py @@ -41,6 +41,8 @@ DATA_DIRECTORY = expanduser('~/Library/Application Support/fman') elif PLATFORM == 'Linux': DATA_DIRECTORY = expanduser('~/.config/fman') +else: + raise NotImplementedError('Unsupported platform: %s' % PLATFORM) class ApplicationCommand: def __init__(self, window): diff --git a/src/main/python/fman/fs.py b/src/main/python/fman/fs.py index 2cf8f00f..d020a0a7 100644 --- a/src/main/python/fman/fs.py +++ b/src/main/python/fman/fs.py @@ -146,7 +146,7 @@ def prepare_trash(self, path): raise self._operation_not_implemented() return [Task( 'Deleting ' + path.rsplit('/', 1)[-1], - fn=self.delete, args=(path,), size=1 + fn=self.move_to_trash, args=(path,), size=1 )] def touch(self, path): raise self._operation_not_implemented() diff --git a/src/main/python/fman/impl/model/worker.py b/src/main/python/fman/impl/model/worker.py index 51066233..df74de27 100644 --- a/src/main/python/fman/impl/model/worker.py +++ b/src/main/python/fman/impl/model/worker.py @@ -21,7 +21,7 @@ def submit(self, priority, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: return - self._queue.put(WorkItem(priority, fn, *args, *kwargs)) + self._queue.put(WorkItem(priority, fn, *args, **kwargs)) def shutdown(self): with self._shutdown_lock: self._shutdown = True @@ -56,7 +56,7 @@ def __lt__(self, other): return NotImplemented def __eq__(self, other): try: - return self._fn, self._args, self._kwargs, self._priority == \ - other._fn, other._args, other._kwargs, other._priority + return (self._fn, self._args, self._kwargs, self._priority) == \ + (other._fn, other._args, other._kwargs, other._priority) except AttributeError: return NotImplemented \ No newline at end of file diff --git a/src/main/python/fman/impl/widgets.py b/src/main/python/fman/impl/widgets.py index c92c08fc..12a8fe24 100644 --- a/src/main/python/fman/impl/widgets.py +++ b/src/main/python/fman/impl/widgets.py @@ -155,7 +155,7 @@ def get_sort_column(self): return column, ascending @run_in_main_thread def get_column_widths(self): - return [self._file_view.columnWidth(i) for i in (0, 1)] + return [self._file_view.columnWidth(i) for i in range(self._model.columnCount())] @run_in_main_thread def set_column_widths(self, column_widths): num_columns = self._model.columnCount() diff --git a/src/main/resources/base/Plugins/Core/core/commands/__init__.py b/src/main/resources/base/Plugins/Core/core/commands/__init__.py index 319553d4..644397e8 100644 --- a/src/main/resources/base/Plugins/Core/core/commands/__init__.py +++ b/src/main/resources/base/Plugins/Core/core/commands/__init__.py @@ -17,7 +17,7 @@ from io import UnsupportedOperation from itertools import chain from os import strerror -from os.path import basename, pardir +from os.path import pardir from pathlib import PurePath from PyQt5.QtCore import QUrl from PyQt5.QtGui import QDesktopServices From 98a8c0eb433e69f52321d8639016feeb0c3ac76f Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sat, 2 May 2026 03:34:28 +0200 Subject: [PATCH 2/7] fix failing tests --- src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py b/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py index 837f2bc2..a68a38d6 100644 --- a/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py +++ b/src/main/resources/base/Plugins/Core/core/tests/fs/test_zip.py @@ -351,7 +351,7 @@ def _read_directory(self, dir_path): child_contents = self._read_directory(child) else: child_contents = child.read_text() - result[child.name] = child_contents + result[normalize('NFC', child.name)] = child_contents return result def _expect_zip_contents(self, contents, zip_file_path): with TemporaryDirectory() as tmp_dir: From 8cf09613cc230d23046428fc31f161b238ac9b3c Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sat, 2 May 2026 03:43:47 +0200 Subject: [PATCH 3/7] all pending plugin errors are now shown at startup instead of just the first one --- src/main/python/fman/impl/plugins/error.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/fman/impl/plugins/error.py b/src/main/python/fman/impl/plugins/error.py index 8b19d965..c6797555 100644 --- a/src/main/python/fman/impl/plugins/error.py +++ b/src/main/python/fman/impl/plugins/error.py @@ -46,8 +46,8 @@ def handle_system_exit(self, code=0): self._app.exit(code) def on_main_window_shown(self, main_window): self._main_window = main_window - if self._pending_error_messages: - self._main_window.show_alert(self._pending_error_messages[0]) + for message in self._pending_error_messages: + self._main_window.show_alert(message) def _get_plugin_traceback(self, exc): if isinstance(exc, ThemeError): return exc.description From 064b5f0eb887c97202e3f80b7973715a33d7aa7d Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sat, 2 May 2026 03:49:46 +0200 Subject: [PATCH 4/7] =?UTF-8?q?=20=20-=20command=5Fregistry.py=20=E2=80=94?= =?UTF-8?q?=20=5Fset=5Fcontext=20now=20uses=20try/finally=20so=20cm.=5F=5F?= =?UTF-8?q?exit=5F=5F=20always=20runs,=20even=20on=20exception.=20=20=20-?= =?UTF-8?q?=20util/qt/=5F=5Finit=5F=5F.py=20=E2=80=94=20Added=20missing=20?= =?UTF-8?q?c=5Fvoid=5Fp=20import=20from=20ctypes,=20fixing=20a=20macOS=20r?= =?UTF-8?q?untime=20crash.=20=20=20-=20table.py=20=E2=80=94=20Fixed=20off-?= =?UTF-8?q?by-one:=20bounds=20check=20now=20rejects=20len=20+=201=20correc?= =?UTF-8?q?tly.=20=20=20-=20widgets.py=20=E2=80=94=20Added=20null=20guard?= =?UTF-8?q?=20on=20=5Fmain=5Fwindow=20before=20accessing=20it=20in=20state?= =?UTF-8?q?=20change=20handler.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/python/fman/impl/model/table.py | 2 +- src/main/python/fman/impl/plugins/command_registry.py | 8 +++++--- src/main/python/fman/impl/util/qt/__init__.py | 3 ++- src/main/python/fman/impl/widgets.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/python/fman/impl/model/table.py b/src/main/python/fman/impl/model/table.py index 33fe1c80..c1d17ff6 100644 --- a/src/main/python/fman/impl/model/table.py +++ b/src/main/python/fman/impl/model/table.py @@ -151,7 +151,7 @@ def insert(self, rows, first_rownum): new_keys = {row.key: first_rownum + i for i, row in enumerate(rows)} with self._lock: # Perform this check here, once we have the lock: - if first_rownum < 0 or first_rownum > len(self._rows) + 1: + if first_rownum < 0 or first_rownum > len(self._rows): raise ValueError('Invalid first_rownum: %d' % first_rownum) num_rows = len(rows) for row in self._rows[first_rownum:]: diff --git a/src/main/python/fman/impl/plugins/command_registry.py b/src/main/python/fman/impl/plugins/command_registry.py index e7204763..7bb1a0e6 100644 --- a/src/main/python/fman/impl/plugins/command_registry.py +++ b/src/main/python/fman/impl/plugins/command_registry.py @@ -153,9 +153,11 @@ def _set_context(self, pane, file_under_cursor=_DEFAULT): if file_under_cursor is not self._DEFAULT: cm = pane._override_file_under_cursor(file_under_cursor) cm.__enter__() - yield - if file_under_cursor is not self._DEFAULT: - cm.__exit__(None, None, None) + try: + yield + finally: + if file_under_cursor is not self._DEFAULT: + cm.__exit__(None, None, None) def _get_default_aliases(cmd_class): return re.sub(r'([a-z])([A-Z])', r'\1 \2', cmd_class.__name__)\ diff --git a/src/main/python/fman/impl/util/qt/__init__.py b/src/main/python/fman/impl/util/qt/__init__.py index 9d77ee96..8baddbad 100644 --- a/src/main/python/fman/impl/util/qt/__init__.py +++ b/src/main/python/fman/impl/util/qt/__init__.py @@ -16,8 +16,9 @@ def disable_window_animations_mac(window): # penalties and leads to subtle changes in behaviour. We therefore wait for # the Show event: def eventFilter(target, event): + from ctypes import c_void_p from objc import objc_object - view = objc_object(c_void_p=int(target.winId())) + view = objc_object(c_void_p=c_void_p(int(target.winId()))) NSWindowAnimationBehaviorNone = 2 view.window().setAnimationBehavior_(NSWindowAnimationBehaviorNone) FilterEventOnce(window, QEvent.Show, eventFilter) diff --git a/src/main/python/fman/impl/widgets.py b/src/main/python/fman/impl/widgets.py index 12a8fe24..5a591d98 100644 --- a/src/main/python/fman/impl/widgets.py +++ b/src/main/python/fman/impl/widgets.py @@ -37,7 +37,7 @@ def exit(self, returnCode=0): def set_style_sheet(self, stylesheet): self.setStyleSheet(stylesheet) def _on_state_changed(self, new_state): - if new_state == Qt.ApplicationActive: + if new_state == Qt.ApplicationActive and self._main_window is not None: for pane in self._main_window.get_panes(): pane.reload() From 93f580e40940ccf1b539a398466bd1b05b1eae11 Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sat, 2 May 2026 03:50:56 +0200 Subject: [PATCH 5/7] =?UTF-8?q?=20=20-=20util/path.py=20=E2=80=94=20normal?= =?UTF-8?q?ize=20now=20loops=20until=20all=20..=20segments=20are=20resolve?= =?UTF-8?q?d,=20so=20a/b/c/../../d=20correctly=20becomes=20a/d.=20=20=20-?= =?UTF-8?q?=20session.py=20=E2=80=94=20Removed=20dead=20=5Fget=5Fstartup?= =?UTF-8?q?=5Fmessage=20method=20(duplicated=20by=20=5Fshow=5Fstartup=5Fme?= =?UTF-8?q?ssages)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/python/fman/impl/session.py | 7 ------- src/main/python/fman/impl/util/path.py | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/python/fman/impl/session.py b/src/main/python/fman/impl/session.py index 58278245..18144e70 100644 --- a/src/main/python/fman/impl/session.py +++ b/src/main/python/fman/impl/session.py @@ -68,13 +68,6 @@ def _show_startup_messages(self, main_window): 'Updated to v%s. ' \ 'Changelog' % self._fman_version main_window.show_status_message(status_message, timeout_secs=5) - def _get_startup_message(self): - previous_version = self._settings.get('fman_version', None) - if not previous_version or previous_version == self._fman_version: - return 'v%s ready.' % self._fman_version - return 'Updated to v%s. ' \ - 'Changelog' \ - % self._fman_version def _init_panes(self, panes, pane_infos, paths_on_cmdline): with ThreadPoolExecutor(max_workers=len(panes)) as executor: futures = [ diff --git a/src/main/python/fman/impl/util/path.py b/src/main/python/fman/impl/util/path.py index b5ffff8a..7105975f 100644 --- a/src/main/python/fman/impl/util/path.py +++ b/src/main/python/fman/impl/util/path.py @@ -34,5 +34,8 @@ def normalize(path_): if path_ == '.': path_ = '' # Resolve a/../b - path_ = re.subn(r'(^|/)([^/]+)/\.\.(?:$|/)', r'\1', path_)[0] + while True: + path_, count = re.subn(r'(^|/)([^/]+)/\.\.(?:$|/)', r'\1', path_) + if not count: + break return path_.rstrip('/') \ No newline at end of file From 74f1ba7cee0e48c29eaab9b999811bc2df97124e Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Sun, 3 May 2026 06:30:22 +0200 Subject: [PATCH 6/7] bugfix --- src/main/resources/base/Plugins/Core/core/commands/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/base/Plugins/Core/core/commands/__init__.py b/src/main/resources/base/Plugins/Core/core/commands/__init__.py index 644397e8..b6c88f53 100644 --- a/src/main/resources/base/Plugins/Core/core/commands/__init__.py +++ b/src/main/resources/base/Plugins/Core/core/commands/__init__.py @@ -1730,7 +1730,7 @@ def on_command(self, command_name, args): except (KeyError, ValueError): return None if scheme == 'file://': - new_scheme = _get_handler_for_archive(basename(path)) + new_scheme = _get_handler_for_archive(basename(url)) if new_scheme: try: if is_dir(url): From bff1834c7a33c32b6c050ecc967058ebb569c7a3 Mon Sep 17 00:00:00 2001 From: Srecko Skocilic Date: Fri, 8 May 2026 20:04:51 +0200 Subject: [PATCH 7/7] Thread safety and cache integrity improvements - Add Cache._lock with generation counter to detect stale query results - Use RLock per-attr in CacheItem for reentrant nested queries - Split CachedIterator locking: _lock for items, _source_lock for source - Always wrap iterdir results in CachedIterator for uniform cache ops - Protect Event callbacks with Lock, snapshot before iteration - Add _tasks_lock to Executor for safe quit/submit races - Use thread-local for _override_file_under_cursor (was monkey-patching) - Snapshot _listeners list before iteration in DirectoryPane._broadcast - Lock notify_file_changed callback snapshot in FileSystem - Add _cache_lock to IconProvider for thread-safe surrogate creation - Cap _already_visited set at 512, convert recursion to iteration - Add shutdown() to SortedFileSystemModel for clean signal disconnect - Replace assert with RuntimeError in diff.py for production safety - Fix missing _m_files update in record_files.py for filtered entries - Change Task.Canceled base from KeyboardInterrupt to Exception - Add APPDATA null guard on Windows - 47 new concurrency and cache invalidation tests --- src/main/python/fman/__init__.py | 33 +-- src/main/python/fman/fs.py | 9 +- src/main/python/fman/impl/fs_cache.py | 84 ++++++-- src/main/python/fman/impl/model/__init__.py | 31 ++- src/main/python/fman/impl/model/diff.py | 15 +- .../python/fman/impl/model/icon_provider.py | 19 +- .../python/fman/impl/model/record_files.py | 1 + .../python/fman/impl/plugins/mother_fs.py | 81 ++++--- src/main/python/fman/impl/util/__init__.py | 12 +- src/main/python/fman/impl/util/qt/thread.py | 28 +-- .../impl/plugins/test_mother_fs.py | 181 +++++++++++++++- .../python/fman_unittest/impl/test_fs.py | 47 ++++ .../fman_unittest/impl/test_fs_cache.py | 203 +++++++++++++++++- 13 files changed, 642 insertions(+), 102 deletions(-) create mode 100644 src/unittest/python/fman_unittest/impl/test_fs.py diff --git a/src/main/python/fman/__init__.py b/src/main/python/fman/__init__.py index eefe7c12..6470afaa 100644 --- a/src/main/python/fman/__init__.py +++ b/src/main/python/fman/__init__.py @@ -6,6 +6,7 @@ from os import getenv from os.path import join, expanduser from PyQt5.QtWidgets import QMessageBox +from threading import local as _thread_local import re @@ -36,7 +37,10 @@ PLATFORM = platform.name() if PLATFORM == 'Windows': - DATA_DIRECTORY = join(getenv('APPDATA'), 'fman') + _appdata = getenv('APPDATA') + if not _appdata: + raise RuntimeError('APPDATA environment variable is not set') + DATA_DIRECTORY = join(_appdata, 'fman') elif PLATFORM == 'Mac': DATA_DIRECTORY = expanduser('~/Library/Application Support/fman') elif PLATFORM == 'Linux': @@ -57,7 +61,7 @@ def aliases(self): def _set_path_onerror(e, url): if isinstance(e, FileNotFoundError): return dirname(url) - raise + raise e class DirectoryPane: def __init__(self, window, widget, command_registry): @@ -65,12 +69,12 @@ def __init__(self, window, widget, command_registry): self._widget = widget self._command_registry = command_registry self._listeners = [] - self._get_file_under_cursor_orig = self.get_file_under_cursor + self._file_under_cursor_override = _thread_local() def _add_listener(self, listener): self._listeners.append(listener) def _broadcast(self, event, *args): - for listener in self._listeners: + for listener in list(self._listeners): getattr(listener, event)(*args) def get_commands(self): @@ -79,7 +83,7 @@ def run_command(self, name, args=None): if args is None: args = {} while True: - for listener in self._listeners: + for listener in list(self._listeners): rewritten = listener.on_command(name, args) if rewritten: name, args = rewritten @@ -100,6 +104,9 @@ def _remove_filter(self, filter_): def get_selected_files(self): return self._widget.get_selected_files() def get_file_under_cursor(self): + override = getattr(self._file_under_cursor_override, 'value', None) + if override is not None: + return override return self._widget.get_file_under_cursor() def move_cursor_down(self, toggle_selection=False): self._widget.move_cursor_down(toggle_selection) @@ -115,14 +122,12 @@ def move_cursor_page_up(self, toggle_selection=False): self._widget.move_cursor_page_up(toggle_selection) def place_cursor_at(self, file_url): self._widget.place_cursor_at(file_url) - # TODO: Rename to get_location() def get_path(self): return self._widget.get_location() - # TODO: Rename to set_location(...) def set_path(self, dir_url, callback=None, onerror=_set_path_onerror): args = dir_url, '', True while True: - for listener in self._listeners: + for listener in list(self._listeners): rewritten = listener.before_location_change(*args) if rewritten and rewritten != args: args = rewritten @@ -156,9 +161,11 @@ def _has_focus(self): return self._widget.hasFocus() @contextmanager def _override_file_under_cursor(self, value): - self.get_file_under_cursor = lambda: value - yield - self.get_file_under_cursor = self._get_file_under_cursor_orig + self._file_under_cursor_override.value = value + try: + yield + finally: + self._file_under_cursor_override.value = None class Window: def __init__(self, widget, panecmd_registry): @@ -199,10 +206,10 @@ def on_doubleclicked(self, file_url): pass def on_name_edited(self, file_url, new_name): pass - # TODO: Rename to after_location_change() def on_path_changed(self): pass def before_location_change(self, url, sort_column='', ascending=True): + """Return (url, sort_column, ascending) to rewrite, or None to allow.""" pass def on_files_dropped(self, file_urls, dest_dir, is_copy_not_move): pass @@ -275,7 +282,7 @@ def unload_plugin(plugin_path): class Task: - class Canceled(KeyboardInterrupt): + class Canceled(Exception): pass def __init__(self, title, size=0, fn=lambda: None, args=(), kwargs=None): diff --git a/src/main/python/fman/fs.py b/src/main/python/fman/fs.py index d020a0a7..b6ee0308 100644 --- a/src/main/python/fman/fs.py +++ b/src/main/python/fman/fs.py @@ -67,6 +67,9 @@ def notify_file_removed(url): _get_mother_fs().notify_file_removed(url) class FileSystem: + """Base class for file system plugins. Set scheme to eg. 'ftp://'. + Methods receive scheme-stripped paths, except copy/move which get full URLs. + """ scheme = '' @@ -108,7 +111,9 @@ def notify_file_added(self, path): def notify_file_removed(self, path): self._file_removed.trigger(self.scheme + path) def notify_file_changed(self, path): - for callback in self._file_changed_callbacks.get(path, []): + with self._file_changed_callbacks_lock: + callbacks = list(self._file_changed_callbacks.get(path, [])) + for callback in callbacks: callback(self.scheme + path) def samefile(self, path1, path2): return self.resolve(path1) == self.resolve(path2) @@ -151,6 +156,7 @@ def prepare_trash(self, path): def touch(self, path): raise self._operation_not_implemented() def copy(self, src_url, dst_url): + """Unlike other methods, receives full URLs (with scheme), not paths.""" raise self._operation_not_implemented() def prepare_copy(self, src_url, dst_url): if self.copy.__func__ is FileSystem.copy: @@ -161,6 +167,7 @@ def prepare_copy(self, src_url, dst_url): fn=self.copy, args=(src_url, dst_url) )] def move(self, src_url, dst_url): + """Unlike other methods, receives full URLs (with scheme), not paths.""" raise self._operation_not_implemented() def prepare_move(self, src_url, dst_url): if self.move.__func__ is FileSystem.move: diff --git a/src/main/python/fman/impl/fs_cache.py b/src/main/python/fman/impl/fs_cache.py index ad665b6c..bd8b8b32 100644 --- a/src/main/python/fman/impl/fs_cache.py +++ b/src/main/python/fman/impl/fs_cache.py @@ -1,37 +1,89 @@ -from collections import defaultdict -from threading import Lock +# Lock ordering (acquire top-to-bottom, never invert): +# Cache._lock +# CacheItem._children_lock (per-node, parent before child) +# CacheItem._attr_locks_lock (per-node) +# CacheItem attr RLock (per-attr) +# Cache._lock must NOT be held when calling compute_value in query(), +# because compute_value may trigger nested cache operations. +from threading import Lock, RLock class Cache: def __init__(self): + self._lock = Lock() self._root = CacheItem() + self._generation = 0 def put(self, path, attr, value): - self._root.update_child(path).put(attr, value) + with self._lock: + self._root.update_child(path).put(attr, value) def get(self, path, attr): - return self._root.get_child(path).get(attr) + with self._lock: + return self._root.get_child(path).get(attr) def query(self, path, attr, compute_value): - return self._root.update_child(path).query(attr, compute_value) + while True: + with self._lock: + gen = self._generation + item = self._root.update_child(path) + result = item.query(attr, compute_value) + with self._lock: + if self._generation == gen: + return result + item.clear_attr(attr) + def mutate(self, path, attr, fn): + with self._lock: + try: + item = self._root.get_child(path) + value = item.get(attr) + except KeyError: + return + fn(value) def clear(self, path): - if not path: - self._root = CacheItem() - else: + with self._lock: + self._generation += 1 + if not path: + self._root = CacheItem() + else: + try: + self._root.delete_child(path) + except KeyError: + pass + def clear_attr(self, path, attr): + with self._lock: try: - self._root.delete_child(path) + item = self._root.get_child(path) except KeyError: - pass + return + item.clear_attr(attr) class CacheItem: def __init__(self): self._children = {} + self._children_lock = Lock() self._attrs = {} - self._attr_locks = defaultdict(Lock) + self._attr_locks_lock = Lock() + self._attr_locks = {} def put(self, attr, value): - self._attrs[attr] = value + with self._attr_locks_lock: + if attr not in self._attr_locks: + self._attr_locks[attr] = RLock() + lock = self._attr_locks[attr] + with lock: + self._attrs[attr] = value def get(self, attr): return self._attrs[attr] + def clear_attr(self, attr): + with self._attr_locks_lock: + lock = self._attr_locks.pop(attr, None) + if lock: + with lock: + self._attrs.pop(attr, None) + else: + self._attrs.pop(attr, None) def query(self, attr, compute_value): - # Because `defaultdict` and `Lock` are implemented in C, they do not - # release the GIL and the dict access is atomic: - with self._attr_locks[attr]: + with self._attr_locks_lock: + if attr not in self._attr_locks: + self._attr_locks[attr] = RLock() + lock = self._attr_locks[attr] + with lock: try: return self._attrs[attr] except KeyError: @@ -57,4 +109,4 @@ def delete_child(self, path): if len(parts) == 1: del self._children[parts[0]] else: - self._children[parts[0]].delete_child(parts[1]) \ No newline at end of file + self._children[parts[0]].delete_child(parts[1]) diff --git a/src/main/python/fman/impl/model/__init__.py b/src/main/python/fman/impl/model/__init__.py index 002a15e7..464ebb4a 100644 --- a/src/main/python/fman/impl/model/__init__.py +++ b/src/main/python/fman/impl/model/__init__.py @@ -20,6 +20,8 @@ class SortedFileSystemModel(QSortFilterProxyModel): sort_order_changed = pyqtSignal(int, int) transaction_ended = pyqtSignal() + _MAX_VISITED = 512 + def __init__(self, parent, fs, null_location): super().__init__(parent) self._fs = fs @@ -105,6 +107,9 @@ def _set_location_main( ) self.setSourceModel(new_model) self._connect_signals(new_model) + if len(self._already_visited) > self._MAX_VISITED: + self._already_visited.clear() + self._already_visited.add(url) self._already_visited.add(url) self.location_changed.emit(url) order = Qt.AscendingOrder if ascending else Qt.DescendingOrder @@ -146,21 +151,19 @@ def url(self, index): def find(self, url): return self.mapFromSource(self.sourceModel().find(url)) def _on_file_removed(self, url): + if not self.sourceModel(): + return if is_pardir(url, self.get_location()): - dir_ = dirname(url) - if dir_ == url: - self.set_location(self._null_location) - else: + while True: + dir_ = dirname(url) + if dir_ == url: + self.set_location(self._null_location) + return try: self.set_location(dir_) + return except OSError: - # In a perfect world, would like to only handle - # FileNotFoundError here. But there can of course also be - # other reasons. For example, when on a network share on - # Windows, we may get a PermissionError trying to list a - # parent directory we don't have access to. So catch all - # OSErrors and in the worst case go to null://. - self._on_file_removed(dir_) + url = dir_ def _connect_signals(self, model): # Would prefer signal.connect(self.signal.emit) here. But PyQt doesn't # support it. So we need Python wrappers "_emit_...": @@ -189,5 +192,11 @@ def _emit_sort_order_changed(self, column, order): self.sort_order_changed.emit(column, order) def _emit_transaction_ended(self): self.transaction_ended.emit() + def shutdown(self): + self._fs.file_removed.remove_callback(self._on_file_removed) + model = self.sourceModel() + if model: + self._disconnect_signals(model) + model.shutdown() def __str__(self): return '<%s: %s>' % (self.__class__.__name__, self.get_location()) \ No newline at end of file diff --git a/src/main/python/fman/impl/model/diff.py b/src/main/python/fman/impl/model/diff.py index aa97d82c..f420d95e 100644 --- a/src/main/python/fman/impl/model/diff.py +++ b/src/main/python/fman/impl/model/diff.py @@ -53,7 +53,8 @@ def _move_row(self, src, dest): def _update_row(self, i, row): self._result.append(DiffEntry(i, i + 1, i, [row])) self._old_rows[i] = row - assert self._key_fn(row) == self._old_keys[i] + if self._key_fn(row) != self._old_keys[i]: + raise RuntimeError('Key mismatch after update') def join(diff_entries): if not diff_entries: @@ -135,8 +136,10 @@ def apply(self, insert, move, update, remove): def _type(self): if self._does_cut: if self.rows: - assert len(self.rows) == self.cut_end - self.cut_start - assert self.cut_start == self.insert_start + if len(self.rows) != self.cut_end - self.cut_start: + raise RuntimeError('Row count mismatch in update entry') + if self.cut_start != self.insert_start: + raise RuntimeError('cut_start != insert_start in update entry') return 'update' else: if self.insert_start != -1: @@ -144,8 +147,10 @@ def _type(self): else: return 'remove' else: - assert self.rows - assert self.insert_start != -1 + if not self.rows: + raise RuntimeError('Insert entry has no rows') + if self.insert_start == -1: + raise RuntimeError('Insert entry has no insert_start') return 'insert' @property def _does_cut(self): diff --git a/src/main/python/fman/impl/model/icon_provider.py b/src/main/python/fman/impl/model/icon_provider.py index 482a794c..e389e95d 100644 --- a/src/main/python/fman/impl/model/icon_provider.py +++ b/src/main/python/fman/impl/model/icon_provider.py @@ -5,6 +5,7 @@ from PyQt5.QtCore import QFileInfo from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QFileIconProvider +from threading import Lock import logging import sys @@ -21,6 +22,7 @@ def __init__(self, qt_icon_provider, fs, cache_dir): f.suffix: self._get_qt_icon(f) for f in Path(cache_dir).glob('file*') } + self._cache_lock = Lock() def get_icon(self, url): scheme, path = splitscheme(url) if scheme == 'file://': @@ -32,14 +34,13 @@ def get_icon(self, url): if self._fs.is_dir(url): return self._folder_icon suffix = PurePosixPath(path).suffix - if suffix not in self._cache: - surrogate = Path(self._cache_dir, 'file' + suffix) - with surrogate.open('w') as f: - # At least Gnome doesn't display a proper icon unless the file - # has some contents. So give it some: - f.write('fman') - self._cache[suffix] = self._get_qt_icon(surrogate) - return self._cache[suffix] + with self._cache_lock: + if suffix not in self._cache: + surrogate = Path(self._cache_dir, 'file' + suffix) + with surrogate.open('w') as f: + f.write('fman') + self._cache[suffix] = self._get_qt_icon(surrogate) + return self._cache[suffix] def _get_qt_icon(self, path): if not isinstance(path, str): path = str(path) @@ -105,7 +106,7 @@ def _query_gio_info(self, file_path, *args): try: return gio_file.query_info(*args) except self.GLib.GError as e: - if e.message and e.message.endswith('No such file or directory'): + if str(e).endswith('No such file or directory'): raise filenotfounderror(file_path) else: raise diff --git a/src/main/python/fman/impl/model/record_files.py b/src/main/python/fman/impl/model/record_files.py index a4d92d15..8e0117c4 100644 --- a/src/main/python/fman/impl/model/record_files.py +++ b/src/main/python/fman/impl/model/record_files.py @@ -92,6 +92,7 @@ def _preprocess_existing(self): else: if not self._m_accepts(file_): to_remove.append(rownum) + self._m_files[file_.url] = file_ continue if file_ != old_file: to_update.append((rownum, file_)) diff --git a/src/main/python/fman/impl/plugins/mother_fs.py b/src/main/python/fman/impl/plugins/mother_fs.py index 552b4f59..2f975abb 100644 --- a/src/main/python/fman/impl/plugins/mother_fs.py +++ b/src/main/python/fman/impl/plugins/mother_fs.py @@ -51,9 +51,9 @@ def iterdir(self, url): child, path = self._split(url) def compute_value(): iterator = getattr(child, 'iterdir')(path) - if hasattr(iterator, '__next__'): - iterator = CachedIterator(iterator) - return iterator + if not hasattr(iterator, '__next__'): + iterator = iter(iterator) + return CachedIterator(iterator) return child.cache.query(path, 'iterdir', compute_value) def query(self, url, fs_method_name): child, path = self._split(url) @@ -170,6 +170,7 @@ def clear_cache(self, url): child.cache.clear(path) def notify_file_added(self, url): child, path = self._split(url) + child.cache.clear(path) child.notify_file_added(path) def notify_file_changed(self, url): child, path = self._split(url) @@ -179,9 +180,11 @@ def notify_file_removed(self, url): child.notify_file_removed(path) def _on_file_added(self, url): self._add_to_parent(url) + self._clear_parent_stat(url) self.file_added.trigger(url) def _on_file_removed(self, url): self._remove(url) + self._clear_parent_stat(url) self.file_removed.trigger(url) def _split(self, url): scheme, path = splitscheme(url) @@ -194,58 +197,84 @@ def _remove(self, url): child, path = self._split(url) child.cache.clear(path) parent_path = splitscheme(dirname(url))[1] + name = basename(url) + child.cache.mutate(parent_path, 'iterdir', lambda ci: ci.remove(name)) + def _clear_parent_stat(self, url): + parent = dirname(url) try: - parent_files = child.cache.get(parent_path, 'iterdir') - except KeyError: - pass - else: - try: - parent_files.remove(basename(url)) - except ValueError: - pass + child, parent_path = self._split(parent) + except FileNotFoundError: + return + for attr in ('stat', 'icon'): + child.cache.clear_attr(parent_path, attr) def _add_to_parent(self, url): parent = dirname(url) - child, parent_path = self._split(parent) try: - parent_files = child.cache.get(parent_path, 'iterdir') - except KeyError: - pass - else: - parent_files.append(basename(url)) + child, parent_path = self._split(parent) + except FileNotFoundError: + return + name = basename(url) + child.cache.mutate(parent_path, 'iterdir', lambda ci: ci.append(name)) +# Lock ordering within CachedIterator: +# _lock (protects _items list and _item_counts dict) +# _source_lock (serializes source iterator advancement) +# _lock is never held when acquiring _source_lock. Inside _source_lock, +# _lock is briefly re-acquired for snapshots — this is safe because no +# code path holds _lock and then acquires _source_lock. class CachedIterator: def __init__(self, source): self._source = source self._lock = Lock() + self._source_lock = Lock() self._items = [] self._item_counts = {} def remove(self, item): with self._lock: self._record(item, delta=-1) + self._item_counts[item] = min(self._item_counts[item], 0) def append(self, item): # N.B.: Behaves like set#add(...), not like list#append(...)! with self._lock: self._record(item) + if self._item_counts[item] <= 0: + self._item_counts[item] = 1 + if item not in self._items: + self._items.append(item) def __iter__(self): return _CachedIterator(self) def get_next(self, pointer): + original_pointer = pointer with self._lock: - for pointer in range(pointer, len(self._items)): - item = self._items[pointer] + for i in range(pointer, len(self._items)): + item = self._items[i] if self._item_counts[item] > 0: - return pointer + 1, item + return i + 1, item + pointer = len(self._items) + with self._source_lock: while True: + with self._lock: + for p in range(original_pointer, len(self._items)): + item = self._items[p] + if self._item_counts[item] > 0: + return p + 1, item + pointer = len(self._items) value = next(self._source) # Eventually raises StopIteration - if self._record(value): - return len(self._items), value + with self._lock: + if self._record(value): + return len(self._items), value def _record(self, value, delta=1): try: - self._item_counts[value] += delta - return False + old_count = self._item_counts[value] + self._item_counts[value] = old_count + delta + return old_count <= 0 and old_count + delta > 0 except KeyError: + if delta < 0: + self._item_counts[value] = delta + return False self._items.append(value) self._item_counts[value] = delta - return True + return delta > 0 class _CachedIterator: def __init__(self, parent): @@ -255,4 +284,4 @@ def __iter__(self): return self def __next__(self): self._pointer, result = self._parent.get_next(self._pointer) - return result \ No newline at end of file + return result diff --git a/src/main/python/fman/impl/util/__init__.py b/src/main/python/fman/impl/util/__init__.py index 519176cc..64a3fa37 100644 --- a/src/main/python/fman/impl/util/__init__.py +++ b/src/main/python/fman/impl/util/__init__.py @@ -2,6 +2,7 @@ from os import listdir, strerror from os.path import join, basename, expanduser, dirname, realpath, relpath, \ pardir, splitdrive +from threading import Lock import errno import os @@ -63,12 +64,17 @@ def __init__(self, *args): class Event: def __init__(self): self._callbacks = [] + self._lock = Lock() def add_callback(self, callback): - self._callbacks.append(callback) + with self._lock: + self._callbacks.append(callback) def remove_callback(self, callback): - self._callbacks.remove(callback) + with self._lock: + self._callbacks.remove(callback) def trigger(self, *args): - for callback in self._callbacks: + with self._lock: + callbacks = list(self._callbacks) + for callback in callbacks: callback(*args) # Copied from core.util: diff --git a/src/main/python/fman/impl/util/qt/thread.py b/src/main/python/fman/impl/util/qt/thread.py index 64a282dd..f51161f2 100644 --- a/src/main/python/fman/impl/util/qt/thread.py +++ b/src/main/python/fman/impl/util/qt/thread.py @@ -54,23 +54,23 @@ def instance(cls): return cls._INSTANCE def __init__(self, app): self._pending_tasks = [] + self._tasks_lock = Lock() self._app_is_about_to_quit = False app.aboutToQuit.connect(self._about_to_quit) def _about_to_quit(self): - self._app_is_about_to_quit = True - for task in self._pending_tasks: - task.set_exception(SystemExit()) - task.has_run.set() + with self._tasks_lock: + self._app_is_about_to_quit = True + for task in self._pending_tasks: + task.set_exception(SystemExit()) + task.has_run.set() def run_in_thread(self, thread, f, args, kwargs): if QThread.currentThread() == thread: return f(*args, **kwargs) - elif self._app_is_about_to_quit: - # In this case, the target thread's event loop most likely is not - # running any more. This would mean that our task (which is - # submitted to the event loop via signals/slots) is never run. - raise SystemExit() - task = Task(f, args, kwargs) - self._pending_tasks.append(task) + with self._tasks_lock: + if self._app_is_about_to_quit: + raise SystemExit() + task = Task(f, args, kwargs) + self._pending_tasks.append(task) try: receiver = Receiver(task) receiver.moveToThread(thread) @@ -80,7 +80,8 @@ def run_in_thread(self, thread, f, args, kwargs): task.has_run.wait() return task.result finally: - self._pending_tasks.remove(task) + with self._tasks_lock: + self._pending_tasks.remove(task) class Task: def __init__(self, fn, args, kwargs): @@ -92,7 +93,8 @@ def __init__(self, fn, args, kwargs): def __call__(self): try: self._result = self._fn(*self._args, **self._kwargs) - except Exception as e: + self._exception = None + except BaseException as e: self._exception = e finally: self.has_run.set() diff --git a/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py b/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py index 2d5d170a..927613c9 100644 --- a/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py +++ b/src/unittest/python/fman_unittest/impl/plugins/test_mother_fs.py @@ -3,7 +3,7 @@ from fman.fs import FileSystem, cached from fman.impl.plugins.mother_fs import MotherFileSystem, CachedIterator from fman_unittest.impl.model import StubFileSystem -from threading import Thread, Lock, Event +from threading import Thread, Barrier, Lock, Event from time import sleep from unittest import TestCase @@ -141,6 +141,51 @@ def test_relative_paths(self): self.assertTrue(mother_fs.is_dir('stub://dir')) mother_fs.move('stub://a/b', 'stub://a/../b') self.assertTrue(mother_fs.exists('stub://b')) + def test_file_added_clears_parent_stat(self): + fs = StubFileSystem({ + 'a': {'is_dir': True, 'files': []} + }) + mother_fs = self._create_mother_fs(fs) + mother_fs.is_dir('stub://a') + self.assertIsNotNone(fs.cache.get('a', 'is_dir')) + mother_fs.touch('stub://a/b') + with self.assertRaises(KeyError): + fs.cache.get('a', 'stat') + def test_file_removed_clears_parent_stat(self): + fs = StubFileSystem({ + 'a': {'is_dir': True, 'files': ['b']}, + 'a/b': {} + }) + mother_fs = self._create_mother_fs(fs) + mother_fs.is_dir('stub://a') + mother_fs.delete('stub://a/b') + with self.assertRaises(KeyError): + fs.cache.get('a', 'stat') + def test_iterdir_returns_cached_iterator(self): + fs = StubFileSystem({ + 'a': {'is_dir': True, 'files': ['b', 'c']} + }) + mother_fs = self._create_mother_fs(fs) + result = mother_fs.iterdir('stub://a') + self.assertIsInstance(result, CachedIterator) + def test_iterdir_cached_is_same_instance(self): + fs = StubFileSystem({ + 'a': {'is_dir': True, 'files': ['b']} + }) + mother_fs = self._create_mother_fs(fs) + first = mother_fs.iterdir('stub://a') + second = mother_fs.iterdir('stub://a') + self.assertIs(first, second) + def test_iterdir_list_source_becomes_cached_iterator(self): + """Even when underlying FS returns plain list, iterdir wraps it.""" + fs = StubFileSystem({ + 'a': {'is_dir': True, 'files': ['x', 'y']} + }) + mother_fs = self._create_mother_fs(fs) + result = mother_fs.iterdir('stub://a') + self.assertIsInstance(result, CachedIterator) + self.assertEqual(Counter(['x', 'y']), Counter(list(result))) + self.assertEqual(Counter(['x', 'y']), Counter(list(result))) def _create_mother_fs(self, fs): result = MotherFileSystem(None) result.add_child(fs.scheme, fs) @@ -264,6 +309,138 @@ def test_add_remove_out_of_order(self): iterable.remove(1) with self.assertRaises(StopIteration): next(iterator) + def test_remove_then_append_same_item(self): + iterable = CachedIterator(self._generate(1, 2)) + iterator = iter(iterable) + self.assertEqual(1, next(iterator)) + iterable.remove(2) + iterable.append(2) + self.assertEqual(2, next(iterator)) + with self.assertRaises(StopIteration): + next(iterator) + self.assertEqual([1, 2], list(iterable)) + def test_remove_before_yield_then_append(self): + iterable = CachedIterator(self._generate(1, 2, 3)) + iterable.remove(3) + iterable.append(3) + result = list(iterable) + self.assertIn(3, result) + self.assertEqual(Counter([1, 2, 3]), Counter(result)) + def test_pre_remove_unknown_item_then_source_yields_it(self): + iterable = CachedIterator(self._generate(1, 2)) + iterable.remove(2) + result = list(iterable) + self.assertEqual([1], result) + def test_concurrent_remove_append(self): + iterable = CachedIterator(self._generate(*range(100))) + results_before = [] + results_after = [] + barrier = Event() + def reader(): + barrier.wait() + results_after.extend(list(iterable)) + for i in range(50, 100): + iterable.remove(i) + for i in range(50, 100): + iterable.append(i) + results_before = list(iterable) + self.assertEqual(Counter(range(100)), Counter(results_before)) + def test_get_next_pointer_after_all_removed(self): + """get_next must advance pointer past cached items when all are dead.""" + iterable = CachedIterator(self._generate(1, 2, 3, 4)) + iterator = iter(iterable) + self.assertEqual(1, next(iterator)) + self.assertEqual(2, next(iterator)) + iterable.remove(3) + iterable.remove(4) + iterable.append(5) + result = list(iterator) + self.assertIn(5, result) + self.assertNotIn(3, result) + self.assertNotIn(4, result) + def test_concurrent_iterate_and_mutate(self): + """Multiple readers + one mutator must not crash or produce dupes.""" + items = list(range(50)) + iterable = CachedIterator(self._generate(*items)) + errors = [] + results = [None, None] + barrier = Barrier(3) + def reader(idx): + barrier.wait() + try: + results[idx] = list(iterable) + except Exception as e: + errors.append(e) + def mutator(): + barrier.wait() + for i in range(50, 80): + iterable.append(i) + for i in range(10): + iterable.remove(i) + threads = [ + Thread(target=reader, args=(0,)), + Thread(target=reader, args=(1,)), + Thread(target=mutator) + ] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual([], errors) + for result in results: + if result is not None: + self.assertEqual(len(result), len(set(result)), + 'Duplicates found: %r' % result) + def test_concurrent_remove_append_stress(self): + """Hammer remove+append from many threads, no crash or lost items.""" + iterable = CachedIterator(self._generate(*range(200))) + list(iterable) + errors = [] + barrier = Barrier(4) + def remove_thread(start, end): + barrier.wait() + for i in range(start, end): + try: + iterable.remove(i) + except Exception as e: + errors.append(e) + def append_thread(start, end): + barrier.wait() + for i in range(start, end): + try: + iterable.append(i) + except Exception as e: + errors.append(e) + threads = [ + Thread(target=remove_thread, args=(0, 100)), + Thread(target=remove_thread, args=(100, 200)), + Thread(target=append_thread, args=(0, 100)), + Thread(target=append_thread, args=(100, 200)) + ] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual([], errors) + result = list(iterable) + self.assertEqual(len(result), len(set(result))) + def test_concurrent_iteration_with_slow_source(self): + """Two iterators consuming a slow source must not lose items.""" + items = list(range(20)) + iterable = CachedIterator(self._generate_slowly(*items)) + results = [None, None] + barrier = Barrier(2) + def reader(idx): + barrier.wait() + results[idx] = list(iterable) + t1 = Thread(target=reader, args=(0,)) + t2 = Thread(target=reader, args=(1,)) + t1.start() + t2.start() + t1.join() + t2.join() + for result in results: + self.assertEqual(sorted(items), sorted(result)) def _generate(self, *args): yield from args def _generate_slowly(self, *args): @@ -272,4 +449,4 @@ def _generate_slowly(self, *args): yield arg def _consume(self, started, iterable): started.set() - return list(iterable) \ No newline at end of file + return list(iterable) diff --git a/src/unittest/python/fman_unittest/impl/test_fs.py b/src/unittest/python/fman_unittest/impl/test_fs.py new file mode 100644 index 00000000..88921dcc --- /dev/null +++ b/src/unittest/python/fman_unittest/impl/test_fs.py @@ -0,0 +1,47 @@ +from fman.fs import FileSystem +from threading import Thread, Barrier, Lock +from unittest import TestCase + +class NotifyFileChangedTest(TestCase): + def test_callback_called(self): + fs = FileSystem() + results = [] + fs._file_changed_callbacks['test'] = [lambda url: results.append(url)] + fs.notify_file_changed('test') + self.assertEqual([fs.scheme + 'test'], results) + def test_no_callbacks_for_path(self): + fs = FileSystem() + fs.notify_file_changed('nonexistent') + def test_callback_removal_during_notify(self): + fs = FileSystem() + results = [] + callbacks = [] + def remove_self(url): + with fs._file_changed_callbacks_lock: + callbacks.remove(remove_self) + results.append('removed') + def second(url): + results.append('second') + callbacks.extend([remove_self, second]) + fs._file_changed_callbacks['test'] = callbacks + fs.notify_file_changed('test') + self.assertEqual(['removed', 'second'], results) + def test_thread_safety(self): + fs = FileSystem() + call_count = 0 + count_lock = Lock() + def counter(url): + nonlocal call_count + with count_lock: + call_count += 1 + fs._file_changed_callbacks['test'] = [counter] + barrier = Barrier(10) + def notify(): + barrier.wait() + fs.notify_file_changed('test') + threads = [Thread(target=notify) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(10, call_count) diff --git a/src/unittest/python/fman_unittest/impl/test_fs_cache.py b/src/unittest/python/fman_unittest/impl/test_fs_cache.py index 55564880..8da805d7 100644 --- a/src/unittest/python/fman_unittest/impl/test_fs_cache.py +++ b/src/unittest/python/fman_unittest/impl/test_fs_cache.py @@ -1,5 +1,6 @@ -from fman.impl.fs_cache import Cache -from threading import Thread +from fman.impl.fs_cache import Cache, CacheItem +from collections import defaultdict +from threading import Thread, Barrier, Event, RLock from time import sleep from unittest import TestCase @@ -46,6 +47,202 @@ def compute_value(): thread_1.join() thread_2.join() self.assertEqual(1, len(calls)) + def test_query_reentrant_rlock(self): + outer_called = [] + def compute_outer(): + outer_called.append(1) + self.cache.query('nested', 'attr', lambda: 'inner') + return 'outer' + result = self.cache.query('top', 'val', compute_outer) + self.assertEqual('outer', result) + self.assertEqual('inner', self.cache.get('nested', 'attr')) + def test_concurrent_put_and_clear(self): + errors = [] + barrier = Barrier(2) + def writer(): + barrier.wait() + for i in range(100): + try: + self.cache.put('path/%d' % i, 'attr', i) + except Exception as e: + errors.append(e) + def clearer(): + barrier.wait() + for _ in range(100): + try: + self.cache.clear('') + except Exception as e: + errors.append(e) + t1 = Thread(target=writer) + t2 = Thread(target=clearer) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual([], errors) + def test_concurrent_query_different_paths(self): + results = {} + barrier = Barrier(2) + def query_path(path, value): + barrier.wait() + results[path] = self.cache.query(path, 'attr', lambda: value) + t1 = Thread(target=query_path, args=('a', 1)) + t2 = Thread(target=query_path, args=('b', 2)) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual(1, results['a']) + self.assertEqual(2, results['b']) + def test_clear_attr(self): + self.cache.put('path', 'keep', 'yes') + self.cache.put('path', 'remove', 'no') + self.cache.clear_attr('path', 'remove') + self.assertEqual('yes', self.cache.get('path', 'keep')) + with self.assertRaises(KeyError): + self.cache.get('path', 'remove') + def test_clear_attr_nonexistent_path(self): + self.cache.clear_attr('nonexistent', 'attr') + def test_clear_attr_nonexistent_attr(self): + self.cache.put('path', 'exists', 1) + self.cache.clear_attr('path', 'nope') + self.assertEqual(1, self.cache.get('path', 'exists')) + def test_clear_attr_allows_recompute(self): + self.cache.query('p', 'a', lambda: 'first') + self.assertEqual('first', self.cache.get('p', 'a')) + self.cache.clear_attr('p', 'a') + self.cache.query('p', 'a', lambda: 'second') + self.assertEqual('second', self.cache.get('p', 'a')) + def test_clear_nonexistent_path(self): + self.cache.clear('nonexistent/path') + def test_nested_path_put_get(self): + self.cache.put('a/b/c', 'val', 42) + self.assertEqual(42, self.cache.get('a/b/c', 'val')) + def test_clear_parent_clears_children(self): + self.cache.put('a/b', 'val', 1) + self.cache.clear('a') + with self.assertRaises(KeyError): + self.cache.get('a/b', 'val') + def test_query_retries_on_concurrent_clear(self): + """Generation counter forces retry when clear() races with query.""" + compute_count = [0] + compute_started = Event() + clear_done = Event() + def compute_value(): + compute_count[0] += 1 + if compute_count[0] == 1: + compute_started.set() + clear_done.wait(timeout=2) + return 'val_%d' % compute_count[0] + def do_clear(): + compute_started.wait(timeout=2) + self.cache.clear('') + clear_done.set() + t = Thread(target=do_clear) + t.start() + result = self.cache.query('p', 'a', compute_value) + t.join() + self.assertEqual(2, compute_count[0]) + self.assertEqual('val_2', result) + def test_mutate_safe_during_concurrent_clear(self): + """mutate must not crash when clear removes the item concurrently.""" + errors = [] + barrier = Barrier(2) + def mutator(): + barrier.wait() + for _ in range(100): + self.cache.mutate('p', 'list', lambda v: v.append(0)) + def clearer(): + barrier.wait() + for _ in range(100): + self.cache.clear('p') + self.cache.put('p', 'list', [1, 2, 3]) + self.cache.put('p', 'list', [1, 2, 3]) + t1 = Thread(target=mutator) + t2 = Thread(target=clearer) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual([], errors) + def test_concurrent_clear_attr_and_query(self): + """clear_attr and query on same attr must not corrupt state.""" + errors = [] + barrier = Barrier(2) + def querier(): + barrier.wait() + for i in range(100): + try: + self.cache.query('p', 'a', lambda: i) + except Exception as e: + errors.append(e) + def clearer(): + barrier.wait() + for _ in range(100): + self.cache.clear_attr('p', 'a') + self.cache.put('p', 'a', 'initial') + t1 = Thread(target=querier) + t2 = Thread(target=clearer) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual([], errors) + def test_concurrent_query_clear_query_returns_fresh(self): + """After clear, next query recomputes — never returns stale data from + a generation that was already invalidated.""" + for _ in range(20): + cache = Cache() + stale = [False] + compute_started = Event() + clear_done = Event() + def first_compute(): + compute_started.set() + clear_done.wait(timeout=2) + return 'stale' + def second_compute(): + return 'fresh' + def do_clear(): + compute_started.wait(timeout=2) + cache.clear('') + clear_done.set() + t = Thread(target=do_clear) + t.start() + count = [0] + def compute(): + count[0] += 1 + if count[0] == 1: + return first_compute() + return second_compute() + result = cache.query('p', 'a', compute) + t.join() + self.assertEqual('fresh', result) def setUp(self): super().setUp() - self.cache = Cache() \ No newline at end of file + self.cache = Cache() + +class CacheItemTest(TestCase): + def test_attr_locks_are_reentrant(self): + item = CacheItem() + item.query('test', lambda: 'val') + lock = item._attr_locks['test'] + lock.acquire() + acquired = lock.acquire(blocking=False) + self.assertTrue(acquired) + lock.release() + lock.release() + def test_update_child_creates_path(self): + item = CacheItem() + child = item.update_child('a/b/c') + child.put('key', 'val') + self.assertEqual('val', item.get_child('a/b/c').get('key')) + def test_delete_child_nested(self): + item = CacheItem() + item.update_child('a/b').put('k', 'v') + item.delete_child('a/b') + with self.assertRaises(KeyError): + item.get_child('a/b') + def test_get_child_nonexistent(self): + item = CacheItem() + with self.assertRaises(KeyError): + item.get_child('nope')