Initial improvements to get rid of the performance regression in py3.

Byte buffer concatenations are considerably slower here for some reason.
Also there was no need for the memorybuffer.

Author: Byron

.travis.yml

@@ -1,5 +1,7 @@
 language: python
 python:
+  - 2.4
+  - 2.5
   - 2.6
   - 2.7
   - 3.3

README.md

@@ -25,19 +25,15 @@ For performance critical 64 bit applications, a simplified version of memory map
 
 ## Prerequisites
 
-* Python 2.4, 2.5 or 2.6
+* Python 2.4, 2.5, 2.6, 2.7 or 3.3
 * OSX, Windows or Linux
 
 The package was tested on all of the previously mentioned configurations.
 
-
-
 ## Limitations
 
 * The memory access is read-only by design.
 * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0.
-* It wasn't tested on python 2.7 and 3.x.
-
 
 ## Installing smmap
 
@@ -80,7 +76,6 @@ Issues can be filed on github:
 * https://github.com/Byron/smmap/issues
 
 
-
 ## License Information
 
 *smmap* is licensed under the New BSD License.

doc/source/changes.rst

@@ -2,12 +2,16 @@
 Changelog
 #########
 
+**********
+v0.8.2
+**********
+- Cleaned up code and assured it works sufficiently well with python 3
+
 **********
 v0.8.1
 **********
 - A single bugfix
 
-
 **********
 v0.8.0 
 **********

smmap/buf.py

@@ -79,18 +79,18 @@ def __getslice__(self, i, j):
         else:
             l = j-i                 # total length
             ofs = i
-            # Keeping tokens in a list could possible be faster, but the list
-            # overhead outweighs the benefits (tested) !
-            md = bytes()
+            # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower
+            # in the previous iteration of this code
+            md = list()
             while l:
                 c.use_region(ofs, l)
                 assert c.is_valid()
                 d = c.buffer()[:l]
                 ofs += len(d)
                 l -= len(d)
-                md += d
+                md.append(d)
             #END while there are bytes to read
-            return md
+            return bytes().join(md)
         # END fast or slow path
     #{ Interface
 

smmap/mman.py

@@ -1,13 +1,12 @@
 """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
 from .util import (
-                    MapWindow,
-                    MapRegion,
-                    MapRegionList,
-                    is_64_bit,
-                    align_to_mmap,
-                    string_types,
-                    buffer,
-                )
+        MapWindow,
+        MapRegion,
+        MapRegionList,
+        is_64_bit,
+        string_types,
+        buffer,
+    )
 
 from weakref import ref
 import sys
@@ -261,14 +260,14 @@ class StaticWindowMapManager(object):
     def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize):
         """initialize the manager with the given parameters.
         :param window_size: if -1, a default window size will be chosen depending on 
-            the operating system's architechture. It will internally be quantified to a multiple of the page size
+            the operating system's architecture. It will internally be quantified to a multiple of the page size
             If 0, the window may have any size, which basically results in mapping the whole file at one
         :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions.
-            If 0, a viable default iwll be set dependning on the system's architecture.
+            If 0, a viable default will be set depending on the system's architecture.
             It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate
-        :param max_open_handles: if not maxin, limit the amount of open file handles to the given number.
-            Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, 
-            the manager will free as many handles as posisble"""
+        :param max_open_handles: if not maxint, limit the amount of open file handles to the given number.
+            Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, 
+            the manager will free as many handles as possible"""
         self._fdict = dict()
         self._window_size = window_size
         self._max_memory_size = max_memory_size
@@ -277,15 +276,15 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.
         self._handle_count = 0
 
         if window_size < 0:
-            coeff = 32
+            coeff = 64
             if is_64_bit():
                 coeff = 1024
             #END handle arch
             self._window_size = coeff * self._MB_in_bytes
         # END handle max window size
 
         if max_memory_size == 0:
-            coeff = 512
+            coeff = 1024
             if is_64_bit():
                 coeff = 8192
             #END handle arch

smmap/util.py

@@ -23,8 +23,9 @@
 except NameError:
     # Python 3 has no `buffer`; only `memoryview`
     def buffer(obj, offset, size):
-        return memoryview(obj[offset:offset+size])
-
+        # return memoryview(obj[offset:offset+size])
+        # doing it directly is much faster !
+        return obj[offset:offset+size]
 
 def string_types():
     if sys.version_info[0] >= 3: