Implemented use_region. Let the testing begin. Especially the actual data handling will be interesting, which has to work exclusively through buffer objects. There are plenty of layers between the user and the data, which will always be copied when slicing it (as we have no memoryview). The latter one could be implemented on in case we use 2.7 at some point.

Now, let the testing begin !

Author: Byron

smmap/mman.py

@@ -9,6 +9,7 @@
 
 from exc import RegionCollectionError
 from weakref import proxy
+import sys
 
 __all__ = ["MappedMemoryManager"]
 #{ Utilities
@@ -77,23 +78,138 @@ def assign(self, rhs):
 		self._destroy()
 		self._copy_from(rhs)
 
-	def use_region(self, offset, size):
+	def use_region(self, offset, size, _is_recursive=False):
 		"""Assure we point to a window which allows access to the given offset into the file
 		:param offset: absolute offset in bytes into the file
 		:param size: amount of bytes to map
 		:return: this instance - it should be queried for whether it points to a valid memory region.
 			This is not the case if the mapping failed becaues we reached the end of the file
 		:note: The size actually mapped may be smaller than the given size. If that is the case,
 			either the file has reached its end, or the map was created between two existing regions"""
+		need_region = True
+		man = self._manager
+		size = min(size, man.window_size()) 	# clamp size to window size
 
+		if self._region is not None:
+			if self._region.includes_ofs(offset):
+				need_region = False
+			else:
+				self.unuse_region()
+			# END handle existing region
+		# END check existing region
+		
+		if need_region:
+			# abort on offsets beyond our mapped file's size - currently we are invalid
+			if offset > self.file_size():
+				return self
+			# END handle offset too large
+			
+			existing_region = None
+			for region in self._rlist:
+				if region.includes_ofs(offset):
+					existing_region = region
+					break
+				#END handle existing region
+			#END for each existing region
 
+			if existing_region is None:
+				left = MemoryWindow(0, 0)
+				mid = MemoryWindow(offset, size)
+				right = MemoryWindow(self.file_size(), 0)
+				
+				# we want to honor the max memory size, and assure we have anough
+				# memory available
+				man._collect_lru_region(man.window_size())
+				
+				# we assume the list remains sorted by offset
+				insert_pos = 0
+				len_regions = len(self._rlist)
+				if len_regions == 1:
+					if self._rlist[0].ofs_begin() <= offset:
+						insert_pos = 1
+					#END maintain sort
+				else:
+					# find insert position
+					insert_pos = len_regions
+					for i, region in enumerate(self._rlist):
+						if region.ofs_begin() > offset:
+							insert_pos = i
+							break
+						#END if insert position is correct
+					#END for each region
+				# END obtain insert pos
+				
+				# adjust the actual offset and size values to create the largest 
+				# possible mapping
+				if insert_pos == 0:
+					if len_regions:
+						right = MemoryWindow.from_region(self._rlist[insert_pos])
+					#END adjust right side 
+				else:
+					if insert_pos != len_regions:
+						right = MemoryWindow.from_region(self._rlist[insert_pos])
+					# END adjust right window
+					left = MemoryWindow.from_region(self._rlist[insert_pos - 1])
+				#END adjust surrounding windows
+				
+				mid.extend_left_to(left, man._window_size)
+				mid.extend_right_to(right, man._window_size)
+				mid.align()
+				
+				# it can happen that we align beyond the end of the file
+				if mid.ofs_end() > right.ofs:
+					mid.size = right.ofs - mid.ofs
+				#END readjust size
+				
+				# insert new region at the right offset to keep the order
+				try:
+					if man._handle_count >= man._max_handle_count:
+						raise Exception
+					#END assert own imposed max file handles
+					self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size)
+				except Exception:
+					# apparently we are out of system resources or hit a limit
+					# As many more operations are likely to fail in that condition (
+					# like reading a file from disk, etc) we free up as much as possible
+					# As this invalidates our insert position, we have to recurse here
+					# NOTE: The c++ version uses a linked list to curcumvent this, but
+					# using that in python is probably too slow anyway
+					if _is_recursive:
+						# we already tried this, and still have no success in obtaining 
+						# a mapping. This is an exception, so we propagate it
+						raise
+					#END handle existing recursion
+					man._collect_lru_region(0)
+					return self.use_region(offset, size, True) 
+				#END handle exceptions
+				
+				man._handle_count += 1
+				man._memory_size += self._region.size()
+				self._rlist.insert(insert_pos, self._region)
+			else:
+				self._region = existing_region
+			#END need region handling
+		#END handle acquire region
+		
+		self._region.increment_usage_count()
+		self._ofs = offset - self._region.ofs_begin()
+		self._size = min(size, self._region.ofs_end() - offset)
+		
+		return self
+		
 	def unuse_region(self):
 		"""Unuse the ucrrent region. Does nothing if we have no current region
 		:note: the cursor unuses the region automatically upon destruction. It is recommended
 			to unuse the region once you are done reading from it in persistent cursors as it 
 			helps to free up resource more quickly"""
 		self._region = None
-	
+
+	def buffer(self):
+		"""Return a buffer object which allows access to our memory region from our offset
+		to the window size. Please note that it might be smaller than you requested
+		:note: You can only obtain a buffer if this instance is_valid() !"""
+		return buffer(self._region.buffer(), self._ofs, self._size)
+
 	def is_valid(self):
 		""":return: True if we have a valid and usable region"""
 		return self._region is not None
@@ -136,7 +252,6 @@ def path(self):
 	#} END interface
 
 
-	
 class MappedMemoryManager(object):
 	"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily 
 	obtain additional regions assuring there is no overlap.
@@ -161,13 +276,13 @@ class MappedMemoryManager(object):
 
 	_MB_in_bytes = 1024 * 1024
 
-	def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0):
+	def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint):
 		"""initialize the manager with the given parameters.
 		:param window_size: if 0, a default window size will be chosen depending on 
 			the operating system's architechture. It will internally be quantified to a multiple of the page size
 		:param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions.
 			If 0, a viable default iwll be set dependning on the system's architecture.
-		:param max_open_handles: if not ~0, lmit the amount of open file handles to the given number.
+		:param max_open_handles: if not maxin, limit the amount of open file handles to the given number.
 			Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, 
 			the manager will free as many handles as posisble"""
 		self._fdict = dict()
@@ -193,7 +308,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0):
 			self._max_memory_size = coeff * self._MB_in_bytes
 		#END handle max memory size
 
-	def _collect_one_lru_region(self, size):
+	def _collect_lru_region(self, size):
 		"""Unmap the region which was least-recently used and has no client
 		:param size: size of the region we want to map next (assuming its not already mapped partially or full
 			if 0, we try to free any available region
@@ -253,6 +368,10 @@ def mapped_memory_size(self):
 		""":return: amount of bytes currently mapped in total"""
 		return self._memory_size
 
+	def max_file_handles(self):
+		""":return: maximium amount of handles we may have opened"""
+		return self._max_handle_count
+		
 	def max_mapped_memory_size(self):
 		""":return: maximum amount of memory we may allocate"""
 		return self._max_memory_size

smmap/test/test_mman.py

@@ -12,16 +12,36 @@
 class TestMMan(TestBase):
 
 	def test_cursor(self):
+		fc = FileCreator(self.k_window_test_size, "cursor_test")
+		
 		man = MappedMemoryManager()
-		c = MemoryCursor(man)
-		assert not c.is_valid()
-		assert not c.is_associated()
+		ci = MemoryCursor(man)	# invalid cursor
+		assert not ci.is_valid()
+		assert not ci.is_associated()
+		assert ci.size() == 0		# this is cached, so we can query it in invalid state
+		
+		cv = man.make_cursor(fc.path)
+		assert not cv.is_valid()	# no region mapped yet
+		assert cv.is_associated()# but it know where to map it from
+		assert cv.file_size() == fc.size
+		assert cv.path() == fc.path
 
 		# copy module
+		cio = copy(cv)
+		assert not cio.is_valid() and cio.is_associated()
 
 		# assign method
+		assert not ci.is_associated()
+		ci.assign(cv)
+		assert not ci.is_valid() and ci.is_associated()
 
+		# unuse non-existing region is fine
+		cv.unuse_region()
+		cv.unuse_region()
 
+		# destruction is fine (even multiple times)
+		cv._destroy()
+		MemoryCursor(man)._destroy()
 
 	def test_memory_manager(self):
 		man = MappedMemoryManager()
@@ -33,8 +53,14 @@ def test_memory_manager(self):
 		assert man.page_size() == PAGESIZE
 
 		# collection doesn't raise in 'any' mode
-		man._collect_one_lru_region(0)
+		man._collect_lru_region(0)
 		# doesn't raise if we are within the limit
-		man._collect_one_lru_region(10)
+		man._collect_lru_region(10)
 		# raises outside of limit
-		self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint)
+		self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint)
+		
+		
+		# use a region, verify most basic functionality
+		fc = FileCreator(self.k_window_test_size, "manager_test")
+		c = man.make_cursor(fc.path)
+		assert c.use_region(10, 10).is_valid()

smmap/test/test_util.py

@@ -88,6 +88,8 @@ def test_region_list(self):
 		fc = FileCreator(100, "sample_file")
 		ml = MappedRegionList(fc.path)
 
+		assert ml.client_count() == 1
+		
 		assert len(ml) == 0
 		assert ml.path() == fc.path
 		assert ml.file_size() == fc.size

smmap/util.py

@@ -54,7 +54,10 @@ def ofs_end(self):
 		return self.ofs + self.size
 
 	def align(self):
-		self.ofs = align_to_page(self.ofs, 0)
+		"""Assures the previous window area is contained in the new one"""
+		nofs = align_to_page(self.ofs, 0)
+		self.size += self.ofs - nofs	# keep size constant
+		self.ofs = nofs
 		self.size = align_to_page(self.size, 1)
 
 	def extend_left_to(self, window, max_size):
@@ -123,6 +126,10 @@ def __init__(self, path, ofs, size):
 			os.close(fd)
 		#END close file handle
 
+	def buffer(self):
+		""":return: a sliceable buffer which can be used to access the mapped memory"""
+		return self._mf
+		
 	def ofs_begin(self):
 		""":return: absolute byte offset to the first byte of the mapping"""
 		return self._b
@@ -159,6 +166,9 @@ def size(self):
 
 		def ofs_end(self):
 			return len(self._mf)
+			
+		def buffer(self):
+			return self._mfb
 	#END handle compat layer
 
 
@@ -176,6 +186,10 @@ def __init__(self, path):
 		self._path = path
 		self._file_size = None
 
+	def client_count(self):
+		""":return: amount of clients which hold a reference to this instance"""
+		return getrefcount(self)-3
+		
 	def path(self):
 		""":return: path to file whose regions we manage"""
 		return self._path