Changed names to be more descriptive, hopefully. This opens op the option to implement such a manager differently, without the sliding window mechanics, which would be quite simple and not much better than a map of mmaps in the end

Author: Byron

smmap/buf.py

@@ -1,11 +1,11 @@
 """Module with a simple buffer implementation using the memory manager"""
-from mman import MemoryCursor
+from mman import SlidingCursor
 
 import sys
 
-__all__ = ["MappedMemoryBuffer"]
+__all__ = ["SlidingWindowMapBuffer"]
 
-class MappedMemoryBuffer(object):
+class SlidingWindowMapBuffer(object):
 	"""A buffer like object which allows direct byte-wise object and slicing into 
 	memory of a mapped file. The mapping is controlled by the provided cursor.
 	

smmap/mman.py

@@ -1,25 +1,25 @@
 """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files"""
 from util import (
-					MemoryWindow,
-					MappedRegion,
-					MappedRegionList,
+					MapWindow,
+					MapRegion,
+					MapRegionList,
 					is_64_bit,
 				)
 
 from exc import RegionCollectionError
 from weakref import ref
 import sys
 
-__all__ = ["MappedMemoryManager"]
+__all__ = ["SlidingWindowMapManager"]
 #{ Utilities
 
 #}END utilities
 
-class MemoryCursor(object):
+class SlidingCursor(object):
 	"""Pointer into the mapped region of the memory manager, keeping the current window 
 	alive until it is destroyed.
 	
-	Cursors should not be created manually, but are instead returned by the MappedMemoryManager"""
+	Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager"""
 	__slots__ = ( 
 					'_manager',	# the manger keeping all file regions
 					'_rlist',	# a regions list with regions for our file
@@ -29,8 +29,8 @@ class MemoryCursor(object):
 				)
 
 	#{ Configuration
-	MemoryWindowCls = MemoryWindow
-	MappedRegionCls = MappedRegion
+	MapWindowCls = MapWindow
+	MapRegionCls = MapRegion
 	#} END configuration
 
 	def __init__(self, manager = None, regions = None):
@@ -133,9 +133,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False):
 			#END while bisecting
 
 			if existing_region is None:
-				left = self.MemoryWindowCls(0, 0)
-				mid = self.MemoryWindowCls(offset, size)
-				right = self.MemoryWindowCls(self.file_size(), 0)
+				left = self.MapWindowCls(0, 0)
+				mid = self.MapWindowCls(offset, size)
+				right = self.MapWindowCls(self.file_size(), 0)
 
 				# we want to honor the max memory size, and assure we have anough
 				# memory available
@@ -166,13 +166,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False):
 				# possible mapping
 				if insert_pos == 0:
 					if len_regions:
-						right = self.MemoryWindowCls.from_region(a[insert_pos])
+						right = self.MapWindowCls.from_region(a[insert_pos])
 					#END adjust right side 
 				else:
 					if insert_pos != len_regions:
-						right = self.MemoryWindowCls.from_region(a[insert_pos])
+						right = self.MapWindowCls.from_region(a[insert_pos])
 					# END adjust right window
-					left = self.MemoryWindowCls.from_region(a[insert_pos - 1])
+					left = self.MapWindowCls.from_region(a[insert_pos - 1])
 				#END adjust surrounding windows
 
 				mid.extend_left_to(left, window_size)
@@ -189,7 +189,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False):
 					if man._handle_count >= man._max_handle_count:
 						raise Exception
 					#END assert own imposed max file handles
-					self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
+					self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
 				except Exception:
 					# apparently we are out of system resources or hit a limit
 					# As many more operations are likely to fail in that condition (
@@ -301,7 +301,7 @@ def fd(self):
 	#} END interface
 
 
-class MappedMemoryManager(object):
+class SlidingWindowMapManager(object):
 	"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily 
 	obtain additional regions assuring there is no overlap.
 	Once a certain memory limit is reached globally, or if there cannot be more open file handles 
@@ -315,7 +315,7 @@ class MappedMemoryManager(object):
 		space is full."""
 
 	__slots__ = [
-					'_fdict', 			# mapping of path -> MappedRegionList
+					'_fdict', 			# mapping of path -> MapRegionList
 					'_window_size', 	# maximum size of a window
 					'_max_memory_size',	# maximum amount ofmemory we may allocate
 					'_max_handle_count',		# maximum amount of handles to keep open
@@ -324,7 +324,7 @@ class MappedMemoryManager(object):
 				]
 
 	#{ Configuration
-	MappedRegionListCls = MappedRegionList
+	MapRegionListCls = MapRegionList
 	#} END configuration
 
 	_MB_in_bytes = 1024 * 1024
@@ -411,10 +411,10 @@ def make_cursor(self, path_or_fd):
 			prevents the file to be opened again just for the purpose of mapping it."""
 		regions = self._fdict.get(path_or_fd)
 		if regions is None:
-			regions = self.MappedRegionListCls(path_or_fd)
+			regions = self.MapRegionListCls(path_or_fd)
 			self._fdict[path_or_fd] = regions
 		# END obtain region for path
-		return MemoryCursor(self, regions)
+		return SlidingCursor(self, regions)
 
 	def collect(self):
 		"""Collect all available free-to-collect mapped regions

smmap/test/test_buf.py

@@ -1,6 +1,6 @@
 from lib import TestBase, FileCreator
 
-from smmap.mman import MappedMemoryManager
+from smmap.mman import SlidingWindowMapManager
 from smmap.buf import *
 
 from random import randint
@@ -9,8 +9,8 @@
 import os
 
 
-man_optimal = MappedMemoryManager()
-man_worst_case = MappedMemoryManager(	window_size=TestBase.k_window_test_size/100, 
+man_optimal = SlidingWindowMapManager()
+man_worst_case = SlidingWindowMapManager(	window_size=TestBase.k_window_test_size/100, 
 									max_memory_size=TestBase.k_window_test_size/3, 
 									max_open_handles=15)
 
@@ -21,10 +21,10 @@ def test_basics(self):
 
 		# invalid paths fail upon construction
 		c = man_optimal.make_cursor(fc.path)
-		self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)())			# invalid cursor
-		self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size)		# offset too large
+		self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)())			# invalid cursor
+		self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size)		# offset too large
 
-		buf = MappedMemoryBuffer()												# can create uninitailized buffers
+		buf = SlidingWindowMapBuffer()												# can create uninitailized buffers
 		assert buf.cursor() is None
 
 		# can call end access any time
@@ -71,7 +71,7 @@ def test_basics(self):
 		for item in (fc.path, fd):
 			for manager, man_id in ( (man_optimal, 'optimal'), 
 									(man_worst_case, 'worst case')):
-				buf = MappedMemoryBuffer(manager.make_cursor(item))
+				buf = SlidingWindowMapBuffer(manager.make_cursor(item))
 				assert manager.num_file_handles() == 1
 				for access_mode in range(2):	# single, multi
 					num_accesses_left = max_num_accesses

smmap/test/test_mman.py

@@ -1,7 +1,7 @@
 from lib import TestBase, FileCreator
 
 from smmap.mman import *
-from smmap.mman import MemoryCursor
+from smmap.mman import SlidingCursor
 from smmap.util import align_to_mmap
 from smmap.exc import RegionCollectionError
 
@@ -16,8 +16,8 @@ class TestMMan(TestBase):
 	def test_cursor(self):
 		fc = FileCreator(self.k_window_test_size, "cursor_test")
 
-		man = MappedMemoryManager()
-		ci = MemoryCursor(man)	# invalid cursor
+		man = SlidingWindowMapManager()
+		ci = SlidingCursor(man)	# invalid cursor
 		assert not ci.is_valid()
 		assert not ci.is_associated()
 		assert ci.size() == 0		# this is cached, so we can query it in invalid state
@@ -43,10 +43,10 @@ def test_cursor(self):
 
 		# destruction is fine (even multiple times)
 		cv._destroy()
-		MemoryCursor(man)._destroy()
+		SlidingCursor(man)._destroy()
 
 	def test_memory_manager(self):
-		man = MappedMemoryManager()
+		man = SlidingWindowMapManager()
 		assert man.num_file_handles() == 0
 		assert man.num_open_files() == 0
 		assert man.window_size() > 0
@@ -82,7 +82,7 @@ def test_memman_operation(self):
 
 			# small windows, a reasonable max memory. Not too many regions at once
 			max_num_handles = 15
-			man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles)
+			man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles)
 			c = man.make_cursor(item)
 
 			# still empty (more about that is tested in test_memory_manager()

smmap/test/test_util.py

@@ -8,10 +8,10 @@
 class TestMMan(TestBase):
 
 	def test_window(self):
-		wl = MemoryWindow(0, 1)	 	# left
-		wc = MemoryWindow(1, 1)		# center
-		wc2 = MemoryWindow(10, 5)		# another center
-		wr = MemoryWindow(8000, 50)	# right
+		wl = MapWindow(0, 1)	 	# left
+		wc = MapWindow(1, 1)		# center
+		wc2 = MapWindow(10, 5)		# another center
+		wr = MapWindow(8000, 50)	# right
 
 		assert wl.ofs_end() == 1
 		assert wc.ofs_end() == 2
@@ -56,9 +56,9 @@ def test_region(self):
 		fc = FileCreator(self.k_window_test_size, "window_test")
 		half_size = fc.size / 2
 		rofs = align_to_mmap(4200, False)
-		rfull = MappedRegion(fc.path, 0, fc.size)
-		rhalfofs = MappedRegion(fc.path, rofs, fc.size)
-		rhalfsize = MappedRegion(fc.path, 0, half_size)
+		rfull = MapRegion(fc.path, 0, fc.size)
+		rhalfofs = MapRegion(fc.path, rofs, fc.size)
+		rhalfsize = MapRegion(fc.path, 0, half_size)
 
 		# offsets
 		assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
@@ -88,15 +88,15 @@ def test_region(self):
 		assert rfull.usage_count() == 1
 
 		# window constructor
-		w = MemoryWindow.from_region(rfull)
+		w = MapWindow.from_region(rfull)
 		assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
 
 	def test_region_list(self):
 		fc = FileCreator(100, "sample_file")
 
 		fd = os.open(fc.path, os.O_RDONLY)
 		for item in (fc.path, fd):
-			ml = MappedRegionList(item)
+			ml = MapRegionList(item)
 
 			assert ml.client_count() == 1
 

smmap/util.py

@@ -15,7 +15,7 @@
 from sys import getrefcount
 
 __all__ = [	"align_to_mmap", "is_64_bit",
-			"MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"]
+			"MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]
 
 #{ Utilities
 
@@ -39,7 +39,7 @@ def is_64_bit():
 
 #{ Utility Classes 
 
-class MemoryWindow(object):
+class MapWindow(object):
 	"""Utility type which is used to snap windows towards each other, and to adjust their size"""
 	__slots__ = (
 				'ofs',		# offset into the file in bytes
@@ -51,7 +51,7 @@ def __init__(self, offset, size):
 		self.size = size
 
 	def __repr__(self):
-		return "MemoryWindow(%i, %i)" % (self.ofs, self.size) 
+		return "MapWindow(%i, %i)" % (self.ofs, self.size) 
 
 	@classmethod
 	def from_region(cls, region):
@@ -84,7 +84,7 @@ def extend_right_to(self, window, max_size):
 		self.size = min(self.size + (window.ofs - self.ofs_end()), max_size)
 
 
-class MappedRegion(object):
+class MapRegion(object):
 	"""Defines a mapped region of memory, aligned to pagesizes
 	:note: deallocates used region automatically on destruction"""
 	__slots__ = [
@@ -145,7 +145,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0):
 		#END close file handle
 
 	def __repr__(self):
-		return "MappedRegion<%i, %i>" % (self._b, self.size())
+		return "MapRegion<%i, %i>" % (self._b, self.size())
 
 	#{ Interface
 
@@ -201,15 +201,15 @@ def includes_ofs(self, ofs):
 	#} END interface
 
 
-class MappedRegionList(list):
-	"""List of MappedRegion instances associating a path with a list of regions."""
+class MapRegionList(list):
+	"""List of MapRegion instances associating a path with a list of regions."""
 	__slots__ = (
 				'_path_or_fd', 	# path or file descriptor which is mapped by all our regions
 				'_file_size'		# total size of the file we map
 				)
 
 	def __new__(cls, path):
-		return super(MappedRegionList, cls).__new__(cls)
+		return super(MapRegionList, cls).__new__(cls)
 
 	def __init__(self, path_or_fd):
 		self._path_or_fd = path_or_fd