Changed buffer implementation to use a cursor right away instead of taking a path and a manager. This makes it much more flexible, as it doesn't have to care about the manager anymore, making it easier to use and making clear that it is meant for use with a mapped memory manager implementation.

Author: Byron

smmap/buf.py

@@ -7,35 +7,23 @@
 
 class MappedMemoryBuffer(object):
 	"""A buffer like object which allows direct byte-wise object and slicing into 
-	memory of a mapped file. The mapping is controlled by an underlying memory manager.
-	
-	A buffer, once initialized, stays put on providing access to eactly one path.
-	A custom interface allows you to change paths mid way, and to optimize 
-	the resource usage.
-	
-	Please note that this type is only fully usable if you configure it with the 
-	MappedMemoryManager to use.
+	memory of a mapped file. The mapping is controlled by the provided cursor.
 	
 	The buffer is relative, that is if you map an offset, index 0 will map to the 
-	first byte at your given offset."""
+	first byte at the offset you used during initialization or begin_access"""
 	__slots__ = '_c'		# our cursor
 
-	#{ Configuration
-	# A subclass must provide an instance of a (usually global) MappedMemoryManager
-	manager = None
-	#}END configuration
 
-	def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0):
-		"""Initalize the instance to operate on the given path if given.
-		:param path: if not None, the path to the file you want to access
-			If None, you have call begin_access before using the buffer 
+	def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0):
+		"""Initalize the instance to operate on the given cursor.
+		:param cursor: if not None, the associated cursor to the file you want to access
+			If None, you have call begin_access before using the buffer and provide a cursor 
 		:param offset: absolute offset in bytes
 		:param size: the total size of the mapping. Defaults to the maximum possible size
 		:param flags: Additional flags to be passed to os.open
 		:raise ValueError: if the buffer could not achieve a valid state"""
-		self._c = MemoryCursor(self.manager)
-		assert self.manager is not None, "Require the cls.manager variable to be set in subclass"
-		if path and not self.begin_access(path, offset, size, flags):
+		self._c = cursor
+		if cursor and not self.begin_access(cursor, offset, size, flags):
 			raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds")
 		# END handle offset
 
@@ -75,19 +63,19 @@ def __getslice__(self, i, j):
 		# END fast or slow path
 	#{ Interface
 
-	def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0):
+	def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0):
 		"""Call this before the first use of this instance. The method was already
 		called by the constructor in case sufficient information was provided.
 		
 		For more information no the parameters, see the __init__ method
-		:param path: if path is empty or None the existing path will be used if possible. 
+		:param path: if cursor is None the existing one will be used. 
 		:return: True if the buffer can be used"""
-		if path and (not self._c.is_associated() or self._c.path() != path):
-			self._c = self.manager.make_cursor(path)
-		#END get associated cursor
+		if cursor:
+			self._c = cursor
+		#END update our cursor
 
 		# reuse existing cursors if possible
-		if self._c.is_associated():
+		if self._c is not None and self._c.is_associated():
 			return self._c.use_region(offset, size, flags).is_valid()
 		return False
 
@@ -97,7 +85,9 @@ def end_access(self):
 		resources to be freed.
 		
 		Once you called end_access, you must call begin access before reusing this instance!"""
-		self._c.unuse_region()
+		if self._c is not None:
+			self._c.unuse_region()
+		#END unuse region
 
 	def cursor(self):
 		""":return: the currently set cursor which provides access to the data"""

smmap/test/test_buf.py

@@ -7,33 +7,34 @@
 from time import time
 import sys
 
-class TestBuffer(MappedMemoryBuffer):
-	#{ Configuration
-	manager = MappedMemoryManager()
-	#} END configuration
-	
+
+man_optimal = MappedMemoryManager()
+man_worst_case = MappedMemoryManager(	window_size=TestBase.k_window_test_size/100, 
+									max_memory_size=TestBase.k_window_test_size/3, 
+									max_open_handles=15)
 
 class TestBuf(TestBase):
 
 	def test_basics(self):
-		self.failUnlessRaises(AssertionError, MappedMemoryBuffer)			# needs subclass
 		fc = FileCreator(self.k_window_test_size, "buffer_test")
 
 		# invalid paths fail upon construction
-		self.failUnlessRaises(OSError, TestBuffer, "somefile")				# invalid file
-		self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size)		# offset too large
+		c = man_optimal.make_cursor(fc.path)
+		self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)())			# invalid cursor
+		self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size)		# offset too large
 
-		buf = TestBuffer()												# can create uninitailized buffers
-		assert not buf.cursor().is_valid() and not buf.cursor().is_associated()
+		buf = MappedMemoryBuffer()												# can create uninitailized buffers
+		assert buf.cursor() is None
 
 		# can call end access any time
 		buf.end_access()
 		buf.end_access()
 
 		# begin access can revive it, if the offset is suitable
 		offset = 100
-		assert buf.begin_access(fc.path, fc.size) == False
-		assert buf.begin_access(fc.path, offset) == True
+		assert buf.begin_access(c, fc.size) == False
+		assert buf.begin_access(c, offset) == True
+		assert buf.cursor().is_valid()
 
 		# empty begin access keeps it valid on the same path, but alters the offset
 		assert buf.begin_access() == True
@@ -52,20 +53,19 @@ def test_basics(self):
 		# an empty begin access fixes it up again
 		assert buf.begin_access() == True and buf.cursor().is_valid()
 		del(buf)		# ends access automatically
+		del(c)
 
-		man = TestBuffer.manager
-		assert man.num_file_handles() == 1
+		assert man_optimal.num_file_handles() == 1
 
 		# PERFORMANCE
 		# blast away with rnadom access and a full mapping - we don't want to 
 		# exagerate the manager's overhead, but measure the buffer overhead
 		# We do it once with an optimal setting, and with a worse manager which 
 		# will produce small mappings only !
 		max_num_accesses = 1000
-		for manager, man_id in ( (man, 'optimal'), 
-								(MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')):
-			TestBuffer.manager = manager
-			buf = TestBuffer(fc.path)
+		for manager, man_id in ( (man_optimal, 'optimal'), 
+								(man_worst_case, 'worst case')):
+			buf = MappedMemoryBuffer(manager.make_cursor(fc.path))
 			assert manager.num_file_handles() == 1
 			for access_mode in range(2):	# single, multi
 				num_accesses_left = max_num_accesses