System can now deal with file descriptors as input, but it still requires some more testing. Also fds need to remain open to be usable for new mapped regions

Author: Byron

smmap/mman.py

@@ -54,7 +54,7 @@ def _destroy(self):
 			num_clients = self._rlist.client_count() - 2
 			if num_clients == 0 and len(self._rlist) == 0:
 				# Free all resources associated with the mapped file
-				self._manager._fdict.pop(self._rlist.path())
+				self._manager._fdict.pop(self._rlist.path_or_fd())
 			#END remove regions list from manager
 		#END handle regions
 
@@ -190,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False):
 					if man._handle_count >= man._max_handle_count:
 						raise Exception
 					#END assert own imposed max file handles
-					self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags)
+					self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
 				except Exception:
 					# apparently we are out of system resources or hit a limit
 					# As many more operations are likely to fail in that condition (
@@ -278,9 +278,26 @@ def file_size(self):
 		""":return: size of the underlying file"""
 		return self._rlist.file_size()
 
+	def path_or_fd(self):
+		""":return: path or file decriptor of the underlying mapped file"""
+		return self._rlist.path_or_fd()
+
 	def path(self):
-		""":return: path of the underlying mapped file"""
-		return self._rlist.path()
+		""":return: path of the underlying mapped file
+		:raise ValueError: if attached path is not a path"""
+		if isinstance(self._rlist.path_or_fd(), int):
+			raise ValueError("Path queried although mapping was applied to a file descriptor")
+		# END handle type
+		return self._rlist.path_or_fd()
+		
+	def fd(self):
+		""":return: file descriptor used to create the underlying mapping.
+		:note: it is not required to be valid anymore
+		:raise ValueError: if the mapping was not created by a file descriptor"""
+		if isinstance(self._rlist.path_or_fd(), basestring):
+			return ValueError("File descriptor queried although mapping was generated from path")
+		#END handle type
+		return self._rlist.path_or_fd()
 
 	#} END interface
 
@@ -383,12 +400,18 @@ def _collect_lru_region(self, size):
 		return num_found
 
 	#{ Interface 
-	def make_cursor(self, path):
-		""":return: a cursor pointing to the given path. It can be used to map new regions of the file into memory"""
-		regions = self._fdict.get(path)
+	def make_cursor(self, path_or_fd):
+		""":return: a cursor pointing to the given path or file descriptor. 
+		It can be used to map new regions of the file into memory
+		:note: if a file descriptor is given, it is assumed to be open and valid,
+			but may be closed afterwards. To refer to the same file, you may reuse
+			your existing file descriptor, but keep in mind that new windows can only
+			be mapped as long as it stays valid. This is why the using actual file paths
+			are preferred unless you plan to keep the file descriptor open."""
+		regions = self._fdict.get(path_or_fd)
 		if regions is None:
-			regions = self.MappedRegionListCls(path)
-			self._fdict[path] = regions
+			regions = self.MappedRegionListCls(path_or_fd)
+			self._fdict[path_or_fd] = regions
 		# END obtain region for path
 		return MemoryCursor(self, regions)
 

smmap/test/test_util.py

@@ -2,6 +2,7 @@
 
 from smmap.util import *
 
+import os
 import sys
 
 class TestMMan(TestBase):
@@ -86,13 +87,18 @@ def test_region(self):
 
 	def test_region_list(self):
 		fc = FileCreator(100, "sample_file")
-		ml = MappedRegionList(fc.path)
 
-		assert ml.client_count() == 1
-		
-		assert len(ml) == 0
-		assert ml.path() == fc.path
-		assert ml.file_size() == fc.size
+		fd = os.open(fc.path, os.O_RDONLY)
+		for item in (fc.path, fd):
+			ml = MappedRegionList(item)
+			
+			assert ml.client_count() == 1
+			
+			assert len(ml) == 0
+			assert ml.path_or_fd() == item
+			assert ml.file_size() == fc.size
+		#END handle input
+		os.close(fd)
 
 	def test_util(self):
 		assert isinstance(is_64_bit(), bool)	# just call it

smmap/util.py

@@ -93,9 +93,9 @@ class MappedRegion(object):
 	#END handle additional slot
 
 
-	def __init__(self, path, ofs, size, flags = 0):
+	def __init__(self, path_or_fd, ofs, size, flags = 0):
 		"""Initialize a region, allocate the memory map
-		:param path: path to the file to map
+		:param path_or_fd: path to the file to map, or the opened file descriptor
 		:param ofs: **aligned** offset into the file to be mapped 
 		:param size: if size is larger then the file on disk, the whole file will be
 			allocated the the size automatically adjusted
@@ -105,7 +105,12 @@ def __init__(self, path, ofs, size, flags = 0):
 		self._size = 0
 		self._uc = 0
 
-		fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags)
+		if isinstance(path_or_fd, int):
+			fd = path_or_fd
+		else:
+			fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags)
+		#END handle fd
+		
 		try:
 			kwargs = dict(access=ACCESS_READ, offset=ofs)
 			corrected_size = size
@@ -189,29 +194,33 @@ def includes_ofs(self, ofs):
 class MappedRegionList(list):
 	"""List of MappedRegion instances associating a path with a list of regions."""
 	__slots__ = (
-				'_path', 		# path which is mapped by all our regions
+				'_path_or_fd', 	# path or file descriptor which is mapped by all our regions
 				'_file_size'		# total size of the file we map
 				)
 
 	def __new__(cls, path):
 		return super(MappedRegionList, cls).__new__(cls)
 
-	def __init__(self, path):
-		self._path = path
+	def __init__(self, path_or_fd):
+		self._path_or_fd = path_or_fd
 		self._file_size = None
 
 	def client_count(self):
 		""":return: amount of clients which hold a reference to this instance"""
 		return getrefcount(self)-3
 
-	def path(self):
-		""":return: path to file whose regions we manage"""
-		return self._path
+	def path_or_fd(self):
+		""":return: path or file descriptor we are attached to"""
+		return self._path_or_fd
 
 	def file_size(self):
 		""":return: size of file we manager"""
 		if self._file_size is None:
-			self._file_size = os.stat(self._path).st_size
+			if isinstance(self._path_or_fd, basestring):
+				self._file_size = os.stat(self._path_or_fd).st_size
+			else:
+				self._file_size = os.fstat(self._path_or_fd).st_size
+			#END handle path type
 		#END update file size
 		return self._file_size