Merge branch 'py2n3'

Clode cleanup and performance regression fixes in py3

Author: Byron

.travis.yml

@@ -1,5 +1,8 @@
 language: python
 python:
+  # These versions are unsupported by travis, even though smmap claims to still support these outdated versions
+  # - 2.4
+  # - 2.5
   - 2.6
   - 2.7
   - 3.3

README.md

@@ -25,27 +25,21 @@ For performance critical 64 bit applications, a simplified version of memory map
 
 ## Prerequisites
 
-* Python 2.4, 2.5 or 2.6
+* Python 2.4, 2.5, 2.6, 2.7 or 3.3
 * OSX, Windows or Linux
 
 The package was tested on all of the previously mentioned configurations.
 
-
-
 ## Limitations
 
 * The memory access is read-only by design.
 * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0.
-* It wasn't tested on python 2.7 and 3.x.
-
 
 ## Installing smmap
 
-Its easiest to install smmap using the *easy_install* or *pip*  program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively:
+Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program:
 
 ```bash
-$ easy_install smmap
-# or 
 $ pip install smmap
 ```
 
@@ -80,7 +74,6 @@ Issues can be filed on github:
 * https://github.com/Byron/smmap/issues
 
 
-
 ## License Information
 
 *smmap* is licensed under the New BSD License.

doc/source/changes.rst

@@ -2,12 +2,16 @@
 Changelog
 #########
 
+**********
+v0.8.3
+**********
+- Cleaned up code and assured it works sufficiently well with python 3
+
 **********
 v0.8.1
 **********
 - A single bugfix
 
-
 **********
 v0.8.0 
 **********

doc/source/intro.rst

@@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map
 #############
 Prerequisites
 #############
-* Python 2.4, 2.5 or 2.6
+* Python 2.4, 2.5, 2.6, 2.7 or 3.3
 * OSX, Windows or Linux
 
 The package was tested on all of the previously mentioned configurations.
@@ -32,15 +32,12 @@ Limitations
 ###########
 * The memory access is read-only by design.
 * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0.
-* It wasn't tested on python 2.7 and 3.x.
 
 ################
 Installing smmap
 ################
-Its easiest to install smmap using the *easy_install* or *pip*  program, which is part of the `setuptools`_ or `pip`_ respectively::
+Its easiest to install smmap using the *pip*  program::
     
-    $ easy_install smmap
-    # or 
     $ pip install smmap
     
 As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes.
@@ -75,5 +72,4 @@ License Information
 ###################
 *smmap* is licensed under the New BSD License.
 
-.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools
 .. _pip: http://www.pip-installer.org/en/latest/

setup.py

@@ -10,10 +10,10 @@
 
 import smmap
 
-if os.path.exists("README.rst"):
-    long_description = codecs.open('README.rst', "r", "utf-8").read()
+if os.path.exists("README.md"):
+    long_description = codecs.open('README.md', "r", "utf-8").read()
 else:
-    long_description = "See http://github.com/nvie/smmap/tree/master"
+    long_description = "See http://github.com/Byron/smmap"
 
 setup(
     name="smmap",
@@ -32,8 +32,8 @@
         #"Development Status :: 1 - Planning",
         #"Development Status :: 2 - Pre-Alpha",
         #"Development Status :: 3 - Alpha",
-        "Development Status :: 4 - Beta",
-        #"Development Status :: 5 - Production/Stable",
+        # "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         #"Development Status :: 6 - Mature",
         #"Development Status :: 7 - Inactive",
         "Environment :: Console",
@@ -52,4 +52,6 @@
         "Programming Language :: Python :: 3.4",
     ],
     long_description=long_description,
+    tests_require=('nose', 'nosexcover'),
+    test_suite='nose.collector'
 )

smmap/__init__.py

@@ -3,7 +3,7 @@
 __author__ = "Sebastian Thiel"
 __contact__ = "byronimo@gmail.com"
 __homepage__ = "https://github.com/Byron/smmap"
-version_info = (0, 8, 2)
+version_info = (0, 8, 3)
 __version__ = '.'.join(str(i) for i in version_info)
 
 # make everything available in root package for convenience

smmap/buf.py

@@ -1,6 +1,4 @@
 """Module with a simple buffer implementation using the memory manager"""
-from .mman import WindowCursor
-
 import sys
 
 __all__ = ["SlidingWindowMapBuffer"]
@@ -79,18 +77,18 @@ def __getslice__(self, i, j):
         else:
             l = j-i                 # total length
             ofs = i
-            # Keeping tokens in a list could possible be faster, but the list
-            # overhead outweighs the benefits (tested) !
-            md = bytes()
+            # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower
+            # in the previous iteration of this code
+            md = list()
             while l:
                 c.use_region(ofs, l)
                 assert c.is_valid()
                 d = c.buffer()[:l]
                 ofs += len(d)
                 l -= len(d)
-                md += d
+                md.append(d)
             #END while there are bytes to read
-            return md
+            return bytes().join(md)
         # END fast or slow path
     #{ Interface
 

smmap/mman.py

@@ -1,13 +1,12 @@
 """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
 from .util import (
-                    MapWindow,
-                    MapRegion,
-                    MapRegionList,
-                    is_64_bit,
-                    align_to_mmap,
-                    string_types,
-                    buffer,
-                )
+        MapWindow,
+        MapRegion,
+        MapRegionList,
+        is_64_bit,
+        string_types,
+        buffer,
+    )
 
 from weakref import ref
 import sys
@@ -102,7 +101,7 @@ def use_region(self, offset = 0, size = 0, flags = 0):
         :param flags: additional flags to be given to os.open in case a file handle is initially opened
             for mapping. Has no effect if a region can actually be reused.
         :return: this instance - it should be queried for whether it points to a valid memory region.
-            This is not the case if the mapping failed becaues we reached the end of the file
+            This is not the case if the mapping failed because we reached the end of the file
             
         **Note:**: The size actually mapped may be smaller than the given size. If that is the case,
         either the file has reached its end, or the map was created between two existing regions"""
@@ -138,7 +137,7 @@ def unuse_region(self):
         """Unuse the ucrrent region. Does nothing if we have no current region
         
         **Note:** the cursor unuses the region automatically upon destruction. It is recommended
-        to unuse the region once you are done reading from it in persistent cursors as it 
+        to un-use the region once you are done reading from it in persistent cursors as it 
         helps to free up resource more quickly"""
         self._region = None
         # note: should reset ofs and size, but we spare that for performance. Its not 
@@ -204,7 +203,7 @@ def file_size(self):
         return self._rlist.file_size()
 
     def path_or_fd(self):
-        """:return: path or file decriptor of the underlying mapped file"""
+        """:return: path or file descriptor of the underlying mapped file"""
         return self._rlist.path_or_fd()
 
     def path(self):
@@ -238,12 +237,12 @@ class StaticWindowMapManager(object):
     These clients would have to use a SlidingWindowMapBuffer to hide this fact.
     
     This type will always use a maximum window size, and optimize certain methods to 
-    acomodate this fact"""
+    accommodate this fact"""
 
     __slots__ = [
                     '_fdict',           # mapping of path -> StorageHelper (of some kind
                     '_window_size',     # maximum size of a window
-                    '_max_memory_size', # maximum amount ofmemory we may allocate
+                    '_max_memory_size', # maximum amount of memory we may allocate
                     '_max_handle_count',        # maximum amount of handles to keep open
                     '_memory_size',     # currently allocated memory size
                     '_handle_count',        # amount of currently allocated file handles
@@ -261,14 +260,14 @@ class StaticWindowMapManager(object):
     def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize):
         """initialize the manager with the given parameters.
         :param window_size: if -1, a default window size will be chosen depending on 
-            the operating system's architechture. It will internally be quantified to a multiple of the page size
+            the operating system's architecture. It will internally be quantified to a multiple of the page size
             If 0, the window may have any size, which basically results in mapping the whole file at one
         :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions.
-            If 0, a viable default iwll be set dependning on the system's architecture.
-            It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate
-        :param max_open_handles: if not maxin, limit the amount of open file handles to the given number.
-            Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, 
-            the manager will free as many handles as posisble"""
+            If 0, a viable default will be set depending on the system's architecture.
+            It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate
+        :param max_open_handles: if not maxint, limit the amount of open file handles to the given number.
+            Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, 
+            the manager will free as many handles as possible"""
         self._fdict = dict()
         self._window_size = window_size
         self._max_memory_size = max_memory_size
@@ -277,15 +276,15 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.
         self._handle_count = 0
 
         if window_size < 0:
-            coeff = 32
+            coeff = 64
             if is_64_bit():
                 coeff = 1024
             #END handle arch
             self._window_size = coeff * self._MB_in_bytes
         # END handle max window size
 
         if max_memory_size == 0:
-            coeff = 512
+            coeff = 1024
             if is_64_bit():
                 coeff = 8192
             #END handle arch
@@ -351,8 +350,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive):
                 # As many more operations are likely to fail in that condition (
                 # like reading a file from disk, etc) we free up as much as possible
                 # As this invalidates our insert position, we have to recurse here
-                # NOTE: The c++ version uses a linked list to curcumvent this, but
-                # using that in python is probably too slow anyway
                 if is_recursive:
                     # we already tried this, and still have no success in obtaining 
                     # a mapping. This is an exception, so we propagate it
@@ -563,8 +560,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive):
                 # As many more operations are likely to fail in that condition (
                 # like reading a file from disk, etc) we free up as much as possible
                 # As this invalidates our insert position, we have to recurse here
-                # NOTE: The c++ version uses a linked list to curcumvent this, but
-                # using that in python is probably too slow anyway
                 if is_recursive:
                     # we already tried this, and still have no success in obtaining 
                     # a mapping. This is an exception, so we propagate it