@@ -64,8 +64,9 @@ def test_memory_manager(self):
6464 man ._collect_lru_region (0 )
6565 # doesn't raise if we are within the limit
6666 man ._collect_lru_region (10 )
67- # raises outside of limit
68- self .failUnlessRaises (RegionCollectionError , man ._collect_lru_region , sys .maxint )
67+
68+ # doesn't fail if we overallocate
69+ assert man ._collect_lru_region (sys .maxint ) == 0
6970
7071 # use a region, verify most basic functionality
7172 fc = FileCreator (self .k_window_test_size , "manager_test" )
@@ -92,103 +93,122 @@ def test_memman_operation(self):
9293 fc = FileCreator (self .k_window_test_size , "manager_operation_test" )
9394 data = open (fc .path , 'rb' ).read ()
9495 fd = os .open (fc .path , os .O_RDONLY )
95- for item in (fc .path , fd ):
96- assert len (data ) == fc .size
97-
98- # small windows, a reasonable max memory. Not too many regions at once
99- max_num_handles = 15
100- man = SlidingWindowMapManager (window_size = fc .size / 100 , max_memory_size = fc .size / 3 , max_open_handles = max_num_handles )
101- c = man .make_cursor (item )
102-
103- # still empty (more about that is tested in test_memory_manager()
104- assert man .num_open_files () == 0
105- assert man .mapped_memory_size () == 0
106-
107- base_offset = 5000
108- size = man .window_size () / 2
109- assert c .use_region (base_offset , size ).is_valid ()
110- rr = c .region_ref ()
111- assert rr ().client_count () == 2 # the manager and the cursor and us
112-
113- assert man .num_open_files () == 1
114- assert man .num_file_handles () == 1
115- assert man .mapped_memory_size () == rr ().size ()
116- assert c .size () == size
117- assert c .ofs_begin () == base_offset
118- assert rr ().ofs_begin () == 0 # it was aligned and expanded
119- assert rr ().size () == align_to_mmap (man .window_size (), True ) # but isn't larger than the max window (aligned)
120-
121- assert c .buffer ()[:] == data [base_offset :base_offset + size ]
122-
123- # obtain second window, which spans the first part of the file - it is a still the same window
124- assert c .use_region (0 , size - 10 ).is_valid ()
125- assert c .region_ref ()() == rr ()
126- assert man .num_file_handles () == 1
127- assert c .size () == size - 10
128- assert c .ofs_begin () == 0
129- assert c .buffer ()[:] == data [:size - 10 ]
130-
131- # map some part at the end, our requested size cannot be kept
132- overshoot = 4000
133- base_offset = fc .size - size + overshoot
134- assert c .use_region (base_offset , size ).is_valid ()
135- assert man .num_file_handles () == 2
136- assert c .size () < size
137- assert c .region_ref ()() is not rr () # old region is still available, but has not curser ref anymore
138- assert rr ().client_count () == 1 # only held by manager
139- rr = c .region_ref ()
140- assert rr ().client_count () == 2 # manager + cursor
141- assert rr ().ofs_begin () < c .ofs_begin () # it should have extended itself to the left
142- assert rr ().ofs_end () <= fc .size # it cannot be larger than the file
143- assert c .buffer ()[:] == data [base_offset :base_offset + size ]
144-
145- # unising a region makes the cursor invalid
146- c .unuse_region ()
147- assert not c .is_valid ()
148- # but doesn't change anything regarding the handle count - we cache it and only
149- # remove mapped regions if we have to
150- assert man .num_file_handles () == 2
151-
152- # iterate through the windows, verify data contents
153- # this will trigger map collection after a while
154- max_random_accesses = 5000
155- num_random_accesses = max_random_accesses
156- memory_read = 0
157- st = time ()
158-
159- # cache everything to get some more performance
160- includes_ofs = c .includes_ofs
161- max_mapped_memory_size = man .max_mapped_memory_size ()
162- max_file_handles = man .max_file_handles ()
163- mapped_memory_size = man .mapped_memory_size
164- num_file_handles = man .num_file_handles
165- while num_random_accesses :
166- num_random_accesses -= 1
167- base_offset = randint (0 , fc .size - 1 )
96+ max_num_handles = 15
97+ #small_size =
98+ for mtype , args in ( (StaticWindowMapManager , (0 , fc .size / 3 , max_num_handles )),
99+ (SlidingWindowMapManager , (fc .size / 100 , fc .size / 3 , max_num_handles )),):
100+ for item in (fc .path , fd ):
101+ assert len (data ) == fc .size
102+
103+ # small windows, a reasonable max memory. Not too many regions at once
104+ man = mtype (window_size = args [0 ], max_memory_size = args [1 ], max_open_handles = args [2 ])
105+ c = man .make_cursor (item )
168106
169- # precondition
170- assert max_mapped_memory_size >= mapped_memory_size ()
171- assert max_file_handles >= num_file_handles ()
107+ # still empty (more about that is tested in test_memory_manager()
108+ assert man .num_open_files () == 0
109+ assert man .mapped_memory_size () == 0
110+
111+ base_offset = 5000
112+ # window size is 0 for static managers, hence size will be 0. We take that into consideration
113+ size = man .window_size () / 2
172114 assert c .use_region (base_offset , size ).is_valid ()
173- csize = c .size ()
174- assert c .buffer ()[:] == data [base_offset :base_offset + csize ]
175- memory_read += csize
115+ rr = c .region_ref ()
116+ assert rr ().client_count () == 2 # the manager and the cursor and us
176117
177- assert includes_ofs (base_offset )
178- assert includes_ofs (base_offset + csize - 1 )
179- assert not includes_ofs (base_offset + csize )
180- # END while we should do an access
181- elapsed = max (time () - st , 0.001 ) # prevent zero divison errors on windows
182- mb = float (1000 * 1000 )
183- sys .stderr .write ("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n "
184- % (memory_read / mb , max_random_accesses , type (item ), elapsed , (memory_read / mb )/ elapsed ))
185-
186- # an offset as large as the size doesn't work !
187- assert not c .use_region (fc .size , size ).is_valid ()
188-
189- # collection - it should be able to collect all
190- assert man .num_file_handles ()
191- assert man .collect ()
192- assert man .num_file_handles () == 0
193- #END for each item
118+ assert man .num_open_files () == 1
119+ assert man .num_file_handles () == 1
120+ assert man .mapped_memory_size () == rr ().size ()
121+
122+ #assert c.size() == size # the cursor may overallocate in its static version
123+ assert c .ofs_begin () == base_offset
124+ assert rr ().ofs_begin () == 0 # it was aligned and expanded
125+ if man .window_size ():
126+ assert rr ().size () == align_to_mmap (man .window_size (), True ) # but isn't larger than the max window (aligned)
127+ else :
128+ assert rr ().size () == fc .size
129+ #END ignore static managers which dont use windows and are aligned to file boundaries
130+
131+ assert c .buffer ()[:] == data [base_offset :base_offset + (size or c .size ())]
132+
133+ # obtain second window, which spans the first part of the file - it is a still the same window
134+ nsize = (size or fc .size ) - 10
135+ assert c .use_region (0 , nsize ).is_valid ()
136+ assert c .region_ref ()() == rr ()
137+ assert man .num_file_handles () == 1
138+ assert c .size () == nsize
139+ assert c .ofs_begin () == 0
140+ assert c .buffer ()[:] == data [:nsize ]
141+
142+ # map some part at the end, our requested size cannot be kept
143+ overshoot = 4000
144+ base_offset = fc .size - (size or c .size ()) + overshoot
145+ assert c .use_region (base_offset , size ).is_valid ()
146+ if man .window_size ():
147+ assert man .num_file_handles () == 2
148+ assert c .size () < size
149+ assert c .region_ref ()() is not rr () # old region is still available, but has not curser ref anymore
150+ assert rr ().client_count () == 1 # only held by manager
151+ else :
152+ assert c .size () < fc .size
153+ #END ignore static managers which only have one handle per file
154+ rr = c .region_ref ()
155+ assert rr ().client_count () == 2 # manager + cursor
156+ assert rr ().ofs_begin () < c .ofs_begin () # it should have extended itself to the left
157+ assert rr ().ofs_end () <= fc .size # it cannot be larger than the file
158+ assert c .buffer ()[:] == data [base_offset :base_offset + (size or c .size ())]
159+
160+ # unising a region makes the cursor invalid
161+ c .unuse_region ()
162+ assert not c .is_valid ()
163+ if man .window_size ():
164+ # but doesn't change anything regarding the handle count - we cache it and only
165+ # remove mapped regions if we have to
166+ assert man .num_file_handles () == 2
167+ #END ignore this for static managers
168+
169+ # iterate through the windows, verify data contents
170+ # this will trigger map collection after a while
171+ max_random_accesses = 5000
172+ num_random_accesses = max_random_accesses
173+ memory_read = 0
174+ st = time ()
175+
176+ # cache everything to get some more performance
177+ includes_ofs = c .includes_ofs
178+ max_mapped_memory_size = man .max_mapped_memory_size ()
179+ max_file_handles = man .max_file_handles ()
180+ mapped_memory_size = man .mapped_memory_size
181+ num_file_handles = man .num_file_handles
182+ while num_random_accesses :
183+ num_random_accesses -= 1
184+ base_offset = randint (0 , fc .size - 1 )
185+
186+ # precondition
187+ if man .window_size ():
188+ assert max_mapped_memory_size >= mapped_memory_size ()
189+ #END statics will overshoot, which is fine
190+ assert max_file_handles >= num_file_handles ()
191+ assert c .use_region (base_offset , (size or c .size ())).is_valid ()
192+ csize = c .size ()
193+ assert c .buffer ()[:] == data [base_offset :base_offset + csize ]
194+ memory_read += csize
195+
196+ assert includes_ofs (base_offset )
197+ assert includes_ofs (base_offset + csize - 1 )
198+ assert not includes_ofs (base_offset + csize )
199+ # END while we should do an access
200+ elapsed = max (time () - st , 0.001 ) # prevent zero divison errors on windows
201+ mb = float (1000 * 1000 )
202+ sys .stderr .write ("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n "
203+ % (mtype , memory_read / mb , max_random_accesses , type (item ), elapsed , (memory_read / mb )/ elapsed ))
204+
205+ # an offset as large as the size doesn't work !
206+ assert not c .use_region (fc .size , size ).is_valid ()
207+
208+ # collection - it should be able to collect all
209+ assert man .num_file_handles ()
210+ assert man .collect ()
211+ assert man .num_file_handles () == 0
212+ #END for each item
213+ # END for each manager type
194214 os .close (fd )
0 commit comments