Moved all utility types into their own module

Author: Byron

smmap/mman.py

@@ -1,192 +1,46 @@
 """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files"""
 
-__all__ = ["MappedMemoryManager"]
+__all__ = ["MappedMemoryManager", "MemoryCursor"]
 
-import os
-import sys
-import mmap
-
-from mmap import PAGESIZE
-from sys import getrefcount
-
-#{ Utilities
-
-def align_to_page(num, round_up):
-	"""Align the given integer number to the closest page offset, which usually is 4096 bytes.
-	:param round_up: if True, the next higher multiple of page size is used, otherwise
-		the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0)
-	:return: num rounded to closest page"""
-	res = (num / PAGESIZE) * PAGESIZE;
-	if round_up and (res != num):
-		res += PAGESIZE;
-	#END handle size
-	return res;
-
-#}END utilities
-
-class Window(object):
-	"""Utility type which is used to snap windows towards each other, and to adjust their size"""
-	__slots__ = (
-				'ofs',		# offset into the file in bytes
-				'size'				# size of the window in bytes
+from util import (
+					MemoryWindow,
+					MappedRegion,
+					MappedRegionList,
 				)
 
-	def __init__(self, offset, size):
-		self.ofs = offset
-		self.size = size
-
-	def __repr__(self):
-		return "Window(%i, %i)" % (self.ofs, self.size) 
-
-	@classmethod
-	def from_region(cls, region):
-		""":return: new window from a region"""
-		return cls(region.ofs_begin(), region.size())
-
-	def ofs_end(self):
-		return self.ofs + self.size
 
-	def align(self):
-		self.ofs = align_to_page(self.ofs, 0)
-		self.size = align_to_page(self.size, 1)
-
-	def extend_left_to(self, window, max_size):
-		"""Adjust the offset to start where the given window on our left ends if possible, 
-		but don't make yourself larger than max_size.
-		The resize will assure that the new window still contains the old window area"""
-		rofs = self.ofs - window.ofs_end()
-		nsize = rofs + self.size
-		rofs -= nsize - min(nsize, max_size)
-		self.ofs = self.ofs - rofs
-		self.size += rofs
-
-	def extend_right_to(self, window, max_size):
-		"""Adjust the size to make our window end where the right window begins, but don't
-		get larger than max_size"""
-		self.size = min(self.size + (window.ofs - self.ofs_end()), max_size)
-
-
-class MappedRegion(object):
-	"""Defines a mapped region of memory, aligned to pagesizes
-	:note: deallocates used region automatically on destruction"""
-	__slots__ = [
-					'_b'	, 	# beginning of mapping
-					'_mf',	# mapped memory chunk (as returned by mmap)
-					'_uc',	# total amount of usages
-					'_ms'	# actual size of the mapping
-				]
-	_need_compat_layer = sys.version_info[1] < 6
-	
-	if _need_compat_layer:
-		__slots__.append('_mfb')		# mapped memory buffer to provide offset
-	#END handle additional slot
-		
+class MemoryCursor(object):
+	"""Pointer into the mapped region of the memory manager, keeping the current window 
+	alive until it is destroyed"""
+	__slots__ = ( 
+					'_manager',	# the manger keeping all file regions
+					'_regions',	# a regions list with regions for our file
+					'_region',	# WEAK REF to our current region
+					'_ofs',		# relative offset from the actually mapped area to our start area
+					'_size'		# maximum size we should provide
+				)
 
-	def __init__(self, path, ofs, size):
-		"""Initialize a region, allocate the memory map
-		:param path: path to the file to map
-		:param ofs: **aligned** offset into the file to be mapped 
-		:param size: if size is larger then the file on disk, the whole file will be
-			allocated the the size automatically adjusted
-		:raise Exception: if no memory can be allocated"""
-		self._b = ofs
-		self._uc = 0
-		
-		fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0))
-		try:
-			kwargs = dict(access=mmap.ACCESS_READ, offset=ofs)
-			corrected_size = size
-			sizeofs = ofs
-			if self._need_compat_layer:
-				del(kwargs['offset'])
-				corrected_size += ofs
-				sizeofs = 0
-			# END handle python not supporting offset ! Arg
-			
-			# have to correct size, otherwise (instead of the c version) it will 
-			# bark that the size is too large ... many extra file accesses because
-			# if this ... argh !
-			self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs)
-			
-			if self._need_compat_layer:
-				self._mfb = buffer(self._mf, ofs, size)
-			#END handle buffer wrapping
-		finally:
-			os.close(fd)
-		#END close file handle
-		
-	def ofs_begin(self):
-		""":return: absolute byte offset to the first byte of the mapping"""
-		return self._b
-		
-	def size(self):
-		""":return: total size of the mapped region in bytes"""
-		return len(self._mf)
+	def __init__(self, manager = None, regions = None):
+		self._manager = manager
+		self._regions = regions
+		self._region = region
+		self._ofs = 0
+		self._size = 0
 
-	def ofs_end(self):
-		""":return: Absolute offset to one byte beyond the mapping into the file"""
-		return self._b + self.size()
+	def __del__(self):
+		self._destroy()
 
-	def includes_ofs(self, ofs):
-		""":return: True if the given offset can be read in our mapped region"""
-		return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end())
+	def _destroy(self):
+		"""Destruction code to decrement counters"""
 
-	def client_count(self):
-		""":return: number of clients currently using this region"""
-		# -1: self on stack, -1 self in this method, -1 self in getrefcount
-		return getrefcount(self)-3
+	def _copy_from(self, rhs):
+		"""Copy all data from rhs into this instance, handles usage count"""
 
-	def adjust_client_count(self, ofs):
-		"""Adjust the client count by the given positive or negative offset"""
-		self._nc += ofs
-		
-	def usage_count(self):
-		""":return: amount of usages so far"""
-		return self._uc
-		
-	def adjust_usage_count(self, ofs):
-		"""Adjust the usage count by the given positive or negative offset"""
-		self._uc += ofs
-		
-	# re-define all methods which need offset adjustments in compatibility mode
-	if _need_compat_layer:
-		def size(self):
-			return len(self._mf) - self._b
-			
-		def ofs_end(self):
-			return len(self._mf)
-	#END handle compat layer
+	#{ Interface
 
-
-class MappedRegionList(list):
-	"""List of MappedRegion instances associating a path with a list of regions."""
-	__slots__ = (
-				'_path', 		# path which is mapped by all our regions
-				'_file_size'		# total size of the file we map
-				)
-	
-	def __new__(cls, path):
-		return super(MappedRegionList, cls).__new__(cls)
 
-	def __init__(self, path):
-		self._path = path
-		self._file_size = None
-		
-	def path(self):
-		""":return: path to file whose regions we manage"""
-		return self._path
-		
-	def file_size(self):
-		""":return: size of file we manager"""
-		if self._file_size is None:
-			self._file_size = os.stat(self._path).st_size
-		#END update file size
-		return self._file_size
+	#} END interface
 
-
-class Cursor(object):
-	"""Pointer into the mapped region of the memory manager, keeping the current window 
-	alive until it is destroyed"""
 
 
 class MappedMemoryManager(object):

smmap/test/lib.py

@@ -49,7 +49,7 @@ class TestBase(TestCase):
 	"""Foundation used by all tests"""
 
 	#{ Configuration
-	
+	k_window_test_size = 1000 * 1000 * 8 + 5195
 	#} END configuration
 
 	#{ Overrides

smmap/test/test_mman.py

@@ -1,102 +1,11 @@
 from lib import TestBase, FileCreator
 
 from smmap.mman import *
-from smmap.mman import align_to_page
-from smmap.mman import Window
-from smmap.mman import MappedRegion
-from smmap.mman import MappedRegionList
-from smmap.mman import Cursor
-
-import sys
-import mmap
 
 class TestMMan(TestBase):
 
-	_window_test_size = 1000 * 1000 * 8 + 5195
-	
-	def test_window(self):
-		wl = Window(0, 1)	 	# left
-		wc = Window(1, 1)		# center
-		wc2 = Window(10, 5)		# another center
-		wr = Window(8000, 50)	# right
-		
-		assert wl.ofs_end() == 1
-		assert wc.ofs_end() == 2
-		assert wr.ofs_end() == 8050
-		
-		# extension does nothing if already in place
-		maxsize = 100
-		wc.extend_left_to(wl, maxsize)
-		assert wc.ofs == 1 and wc.size == 1
-		wl.extend_right_to(wc, maxsize)
-		wl.extend_right_to(wc, maxsize)
-		assert wl.ofs == 0 and wl.size == 1
-		
-		# an actual left extension
-		pofs_end = wc2.ofs_end()
-		wc2.extend_left_to(wc, maxsize)
-		assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() 
-		
-		
-		# respects maxsize
-		wc.extend_right_to(wr, maxsize)
-		assert wc.ofs == 1 and wc.size == maxsize
-		wc.extend_right_to(wr, maxsize)
-		assert wc.ofs == 1 and wc.size == maxsize
-		
-		# without maxsize
-		wc.extend_right_to(wr, sys.maxint)
-		assert wc.ofs_end() == wr.ofs and wc.ofs == 1
-		
-		# extend left
-		wr.extend_left_to(wc2, maxsize)
-		wr.extend_left_to(wc2, maxsize)
-		assert wr.size == maxsize
-		
-		wr.extend_left_to(wc2, sys.maxint)
-		assert wr.ofs == wc2.ofs_end()
-		
-		wc.align()
-		assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2
-		
-	def test_region(self):
-		fc = FileCreator(self._window_test_size, "window_test")
-		half_size = fc.size / 2
-		rofs = align_to_page(4200, False)
-		rfull = MappedRegion(fc.path, 0, fc.size)
-		rhalfofs = MappedRegion(fc.path, rofs, fc.size)
-		rhalfsize = MappedRegion(fc.path, 0, half_size)
-		
-		# offsets
-		assert rfull.ofs_begin() == 0 and rfull.size() == fc.size
-		assert rfull.ofs_end() == fc.size	# if this method works, it works always
-		
-		assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs
-		assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
-		
-		assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size)
-		assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint)
-		assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
-		
-		# auto-refcount
-		assert rfull.client_count() == 1
-		rfull2 = rfull
-		assert rfull.client_count() == 2
-		
-		# window constructor
-		w = Window.from_region(rfull)
-		assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end()
-		
-	def test_region_list(self):
-		fc = FileCreator(100, "sample_file")
-		ml = MappedRegionList(fc.path)
-		
-		assert len(ml) == 0
-		assert ml.path() == fc.path
-		assert ml.file_size() == fc.size
-		
 	def test_cursor(self):
-		pass
+		man = MappedMemoryManager()  
 
 	def test_basics(self):
 		pass