implemented plenty of operational testing. It shows that its not yet working properly. Intersting, it was so promising, and went just a little bit too smooth. There we go :)

Author: Byron

smmap/mman.py

@@ -8,7 +8,7 @@
 				)
 
 from exc import RegionCollectionError
-from weakref import proxy
+from weakref import ref
 import sys
 
 __all__ = ["MappedMemoryManager"]
@@ -100,7 +100,7 @@ def use_region(self, offset, size, _is_recursive=False):
 
 		if need_region:
 			# abort on offsets beyond our mapped file's size - currently we are invalid
-			if offset > self.file_size():
+			if offset >= self.file_size():
 				return self
 			# END handle offset too large
 
@@ -222,16 +222,21 @@ def ofs_begin(self):
 		""":return: offset to the first byte pointed to by our cursor"""
 		return self._region.ofs_begin() + self._ofs
 
+	def ofs_end(self):
+		""":return: offset to one past the last available byte"""
+		# unroll method calls for performance !
+		return self._region.ofs_begin() + self._ofs + self._size
+		
 	def size(self):
 		""":return: amount of bytes we point to"""
 		return self._size
 
 	def region_ref(self):
-		""":return: weak proxy to our mapped region.
+		""":return: weak ref to our mapped region.
 		:raise AssertionError: if we have no current region. This is only useful for debugging"""
 		if self._region is None:
 			raise AssertionError("region not set")
-		return proxy(self._region)
+		return ref(self._region)
 
 	def includes_ofs(self, ofs):
 		""":return: True if the given absolute offset is contained in the cursors 

smmap/test/test_mman.py

@@ -2,10 +2,11 @@
 
 from smmap.mman import *
 from smmap.mman import MemoryCursor
-from smmap.util import PAGESIZE
-
+from smmap.util import PAGESIZE, align_to_page
 from smmap.exc import RegionCollectionError
 
+from random import randint
+from time import time
 import sys
 from copy import copy
 
@@ -59,8 +60,101 @@ def test_memory_manager(self):
 		# raises outside of limit
 		self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint)
 
-		
 		# use a region, verify most basic functionality
 		fc = FileCreator(self.k_window_test_size, "manager_test")
 		c = man.make_cursor(fc.path)
 		assert c.use_region(10, 10).is_valid()
+		assert c.ofs_begin() == 10
+		assert c.size() == 10
+		assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:]
+		
+	def test_memman_operation(self):
+		# test more access, force it to actually unmap regions
+		fc = FileCreator(self.k_window_test_size, "manager_operation_test")
+		data = open(fc.path, 'rb').read()
+		assert len(data) == fc.size
+		
+		# small windows, a reasonable max memory. Not too many regions at once
+		man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15)
+		c = man.make_cursor(fc.path)
+		
+		# still empty (more about that is tested in test_memory_manager()
+		assert man.num_open_files() == 0
+		assert man.mapped_memory_size() == 0
+		
+		base_offset = 5000
+		size = man.window_size() / 2
+		assert c.use_region(base_offset, size).is_valid()
+		rr = c.region_ref()
+		assert rr().client_count() == 2	# the manager and the cursor and us
+		
+		assert man.num_open_files() == 1
+		assert man.num_file_handles() == 1
+		assert man.mapped_memory_size() == rr().size()
+		assert c.size() == size
+		assert c.ofs_begin() == base_offset
+		assert rr().ofs_begin() == 0		# it was aligned and expanded
+		assert rr().size() == align_to_page(man.window_size(), True)	# but isn't larger than the max window (aligned)
+		
+		assert c.buffer()[:] == data[base_offset:base_offset+size] 
+		
+		# obtain second window, which spans the first part of the file - it is a still the same window
+		assert c.use_region(0, size-10).is_valid()
+		assert c.region_ref()() == rr()
+		assert man.num_file_handles() == 1
+		assert c.size() == size-10
+		assert c.ofs_begin() == 0
+		assert c.buffer()[:] == data[:size-10]
+		
+		# map some part at the end, our requested size cannot be kept
+		overshoot = 4000
+		base_offset = fc.size - size + overshoot
+		assert c.use_region(base_offset, size).is_valid()
+		assert man.num_file_handles() == 2
+		assert c.size() < size
+		assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore
+		assert rr().client_count() == 1 # only held by manager
+		rr = c.region_ref()
+		assert rr().client_count() == 2 # manager + cursor
+		assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left
+		assert rr().ofs_end() <= fc.size # it cannot be larger than the file
+		assert c.buffer()[:] == data[base_offset:base_offset+size]
+		
+		# unising a region makes the cursor invalid
+		c.unuse_region()
+		assert not c.is_valid()
+		# but doesn't change anything regarding the handle count - we cache it and only 
+		# remove mapped regions if we have to
+		assert man.num_file_handles() == 2
+		
+		# an offset as large as the size doesn't work !
+		assert not c.use_region(fc.size, size).is_valid() 
+
+		# iterate through the windows, verify data contents
+		# this will trigger map collection after a while
+		max_random_accesses = 15000
+		num_random_accesses = max_random_accesses
+		memory_read = 0
+		st = time()
+		
+		while num_random_accesses:
+			num_random_accesses += 1
+			base_offset = randint(0, fc.size - 1)
+			
+			# precondition
+			assert man.max_mapped_memory_size() >= man.mapped_memory_size()
+			assert man.max_file_handles() >= man.num_file_handles()
+			
+			assert c.use_region(base_offset, size).is_valid()
+			assert c.buffer()[:] == data[base_offset:base_offset+size]
+			memory_read += c.size()
+			
+			assert c.includes_ofs(base_offset)
+			assert c.includes_ofs(base_offset+c.size()-1)
+			assert not c.includes_ofs(base_offset+c.size())
+		# END while we should do an access
+		elapsed = time() - st
+		mb = 1000 * 1000
+		sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" 
+						% (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed))
+		

smmap/util.py

@@ -83,7 +83,6 @@ class MappedRegion(object):
 					'_b'	, 	# beginning of mapping
 					'_mf',	# mapped memory chunk (as returned by mmap)
 					'_uc',	# total amount of usages
-					'_ms'	# actual size of the mapping
 					'__weakref__'
 				]
 	_need_compat_layer = sys.version_info[1] < 6