Fixed some mapping issues on windows. Fixed some tests to deal with the very different granularity

Author: Byron

smmap/mman.py

@@ -4,7 +4,6 @@
 					MappedRegion,
 					MappedRegionList,
 					is_64_bit,
-					PAGESIZE
 				)
 
 from exc import RegionCollectionError
@@ -446,10 +445,6 @@ def max_mapped_memory_size(self):
 		""":return: maximum amount of memory we may allocate"""
 		return self._max_memory_size
 
-	def page_size(self):
-		""":return: size of a single memory page in bytes"""
-		return PAGESIZE
-		
 	#} END interface
 
 	#{ Special Purpose Interface

smmap/test/test_buf.py

@@ -97,7 +97,7 @@ def test_basics(self):
 					assert manager.num_file_handles()
 					assert manager.collect()
 					assert manager.num_file_handles() == 0
-					elapsed  = time() - st
+					elapsed  = max(time() - st, 0.001)	# prevent zero division errors on windows
 					mb = float(1000*1000)
 					mode_str = (access_mode and "slice") or "single byte"
 					sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" 

smmap/test/test_mman.py

@@ -2,7 +2,7 @@
 
 from smmap.mman import *
 from smmap.mman import MemoryCursor
-from smmap.util import PAGESIZE, align_to_page
+from smmap.util import align_to_mmap
 from smmap.exc import RegionCollectionError
 
 from random import randint
@@ -52,7 +52,6 @@ def test_memory_manager(self):
 		assert man.window_size() > 0
 		assert man.mapped_memory_size() == 0
 		assert man.max_mapped_memory_size() > 0
-		assert man.page_size() == PAGESIZE
 
 		# collection doesn't raise in 'any' mode
 		man._collect_lru_region(0)
@@ -102,7 +101,7 @@ def test_memman_operation(self):
 			assert c.size() == size
 			assert c.ofs_begin() == base_offset
 			assert rr().ofs_begin() == 0		# it was aligned and expanded
-			assert rr().size() == align_to_page(man.window_size(), True)	# but isn't larger than the max window (aligned)
+			assert rr().size() == align_to_mmap(man.window_size(), True)	# but isn't larger than the max window (aligned)
 
 			assert c.buffer()[:] == data[base_offset:base_offset+size] 
 
@@ -164,7 +163,7 @@ def test_memman_operation(self):
 				assert includes_ofs(base_offset+csize-1)
 				assert not includes_ofs(base_offset+csize)
 			# END while we should do an access
-			elapsed = time() - st
+			elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
 			mb = float(1000 * 1000)
 			sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" 
 							% (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed))

smmap/test/test_util.py

@@ -1,6 +1,7 @@
 from lib import TestBase, FileCreator
 
 from smmap.util import *
+from mmap import ALLOCATIONGRANULARITY
 
 import os
 import sys
@@ -50,12 +51,12 @@ def test_window(self):
 		assert wr.ofs == wc2.ofs_end()
 
 		wc.align()
-		assert wc.ofs == 0 and wc.size == PAGESIZE*2
+		assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True)
 
 	def test_region(self):
 		fc = FileCreator(self.k_window_test_size, "window_test")
 		half_size = fc.size / 2
-		rofs = align_to_page(4200, False)
+		rofs = align_to_mmap(4200, False)
 		rfull = MappedRegion(fc.path, 0, fc.size)
 		rhalfofs = MappedRegion(fc.path, rofs, fc.size)
 		rhalfsize = MappedRegion(fc.path, 0, half_size)
@@ -69,7 +70,13 @@ def test_region(self):
 
 		assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size)
 		assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint)
-		assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
+		# with the values we have, this test only works on windows where an alignment 
+		# size of 4096 is assumed.
+		if sys.platform == 'win32':
+			assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0)
+		else:
+			assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
+		#END handle platforms
 
 		# auto-refcount
 		assert rfull.client_count() == 1
@@ -102,6 +109,6 @@ def test_region_list(self):
 
 	def test_util(self):
 		assert isinstance(is_64_bit(), bool)	# just call it
-		assert align_to_page(1, False) == 0
-		assert align_to_page(1, True) == PAGESIZE
+		assert align_to_mmap(1, False) == 0
+		assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY
 

smmap/util.py

@@ -3,22 +3,22 @@
 import sys
 import mmap
 
-from mmap import PAGESIZE, mmap, ACCESS_READ
+from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ
 from sys import getrefcount
 
-__all__ = [	"align_to_page", "is_64_bit",
-			"MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"]
+__all__ = [	"align_to_mmap", "is_64_bit",
+			"MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"]
 
 #{ Utilities
 
-def align_to_page(num, round_up):
+def align_to_mmap(num, round_up):
 	"""Align the given integer number to the closest page offset, which usually is 4096 bytes.
 	:param round_up: if True, the next higher multiple of page size is used, otherwise
 		the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0)
 	:return: num rounded to closest page"""
-	res = (num / PAGESIZE) * PAGESIZE;
+	res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY;
 	if round_up and (res != num):
-		res += PAGESIZE;
+		res += ALLOCATIONGRANULARITY
 	#END handle size
 	return res;
 
@@ -55,10 +55,10 @@ def ofs_end(self):
 
 	def align(self):
 		"""Assures the previous window area is contained in the new one"""
-		nofs = align_to_page(self.ofs, 0)
+		nofs = align_to_mmap(self.ofs, 0)
 		self.size += self.ofs - nofs	# keep size constant
 		self.ofs = nofs
-		self.size = align_to_page(self.size, 1)
+		self.size = align_to_mmap(self.size, 1)
 
 	def extend_left_to(self, window, max_size):
 		"""Adjust the offset to start where the given window on our left ends if possible,