From 2ae54cf086ba159633e893def6f0613fe4983998 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 19 Mar 2025 15:14:02 +0100 Subject: [PATCH 01/13] change seek to write for new file --- python/ndstorage/ndtiff_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 5fcac31..1e06133 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -170,7 +170,8 @@ def write_image(self, index_key, pixels, metadata, bit_depth='auto'): def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width, bit_depth): if self.file.tell() % 2 == 1: - self.file.seek(self.file.tell() + 1) # Make IFD start on word + #self.file.seek(self.file.tell() + 1) # Make IFD start on word + self.file.write(b'\0') # should be equivalent byte_depth = 1 if isinstance(pixels, bytearray) else 2 bytes_per_image_pixels = self._bytes_per_image_pixels(pixels, rgb) From a03ba0dd78cb6ad4fdc7858050e8dc1d5808193d Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 19 Mar 2025 15:15:32 +0100 Subject: [PATCH 02/13] tiff bit depth 8bit bug fix --- python/ndstorage/ndtiff_file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 1e06133..0548b7a 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -174,6 +174,9 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width self.file.write(b'\0') # should be equivalent byte_depth = 1 if isinstance(pixels, bytearray) else 2 + if bit_depth == 8: + byte_depth = 1 #isinstance doesen't work? + bytes_per_image_pixels = self._bytes_per_image_pixels(pixels, rgb) num_entries = 13 From 645495102b5678748d3f1d4db6634575dc6ef826 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 19 Mar 2025 15:18:29 +0100 Subject: [PATCH 03/13] put image lock to prevent errors when threading --- python/ndstorage/ndtiff_dataset.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index fe5a9dc..d6abb9a 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -44,6 +44,8 @@ def __init__(self, dataset_path=None, file_io: NDTiffFileIO = BUILTIN_FILE_IO, s self.file_io = file_io self._lock = threading.RLock() + self._put_file_lock = threading.Lock() + if writable: self.major_version = MAJOR_VERSION self.minor_version = MINOR_VERSION @@ -166,6 +168,8 @@ def read_metadata(self, channel=None, z=None, time=None, position=None, row=None return self._do_read_metadata(axes) def put_image(self, coordinates, image, metadata): + self._put_file_lock.acquire() + if not self._writable: raise RuntimeError("Cannot write to a read-only dataset") @@ -203,6 +207,8 @@ def put_image(self, coordinates, image, metadata): self._index_file.write(index_data_entry.as_byte_buffer().getvalue()) # remove from pending images del self._write_pending_images[frozenset(coordinates.items())] + + self._put_file_lock.release() def finish(self): if self.current_writer is not None: From b12f64117e78c4fe4b03d0100e7b28f166844f99 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 19 Mar 2025 15:48:47 +0100 Subject: [PATCH 04/13] zlib pixel compression --- python/ndstorage/ndtiff_dataset.py | 20 ++++++++--- python/ndstorage/ndtiff_file.py | 53 +++++++++++++++++++++++------- python/ndstorage/ndtiff_index.py | 8 +++-- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index d6abb9a..b1ca4aa 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -21,7 +21,7 @@ class NDTiffDataset(NDStorageBase, WritableNDStorageAPI): """ def __init__(self, dataset_path=None, file_io: NDTiffFileIO = BUILTIN_FILE_IO, summary_metadata=None, - name=None, writable=False, **kwargs): + name=None, writable=False, pixel_compression = 1, **kwargs): """ Provides access to an NDTiffStorage dataset, either one currently being acquired or one on disk @@ -50,6 +50,10 @@ def __init__(self, dataset_path=None, file_io: NDTiffFileIO = BUILTIN_FILE_IO, s self.major_version = MAJOR_VERSION self.minor_version = MINOR_VERSION self._index_file = None + if pixel_compression in[1,8]: + self._pixel_compression = pixel_compression + else: + raise ValueError("Compression scheme must be 1 (No) or 8 (zlib)") if summary_metadata is not None or writable: # this dataset is either: # - a view of an active acquisition. Image data is being written by code on the Java side @@ -167,12 +171,18 @@ def read_metadata(self, channel=None, z=None, time=None, position=None, row=None return self._do_read_metadata(axes) - def put_image(self, coordinates, image, metadata): + def put_image(self, coordinates, image, metadata, pixel_compression = 0): self._put_file_lock.acquire() if not self._writable: raise RuntimeError("Cannot write to a read-only dataset") + if pixel_compression == 0: + pixel_compression = self._pixel_compression + elif not pixel_compression in [1,8]: + warnings.warn(f"Pixel compression: only 1 (no compression) and 8 (zlib) are supported. Using {self._pixel_compression}.") + pixel_compression = self._pixel_compression + # add to write pending images self._write_pending_images[frozenset(coordinates.items())] = (image, metadata) @@ -188,7 +198,7 @@ def put_image(self, coordinates, image, metadata): filename = 'NDTiffStack.tif' if self.name is not None: filename = self.name + '_' + filename - self.current_writer = SingleNDTiffWriter(self.path, filename, self._summary_metadata) + self.current_writer = SingleNDTiffWriter(self.path, filename, self._summary_metadata, self._pixel_compression) self.file_index += 1 # create the index file self._index_file = open(os.path.join(self.path, "NDTiff.index"), "wb") @@ -197,10 +207,10 @@ def put_image(self, coordinates, image, metadata): filename = 'NDTiffStack_{}.tif'.format(self.file_index) if self.name is not None: filename = self.name + '_' + filename - self.current_writer = SingleNDTiffWriter(self.path, filename, self._summary_metadata) + self.current_writer = SingleNDTiffWriter(self.path, filename, self._summary_metadata, self._pixel_compression) self.file_index += 1 - index_data_entry = self.current_writer.write_image(frozenset(coordinates.items()), image, metadata) + index_data_entry = self.current_writer.write_image(frozenset(coordinates.items()), image, metadata, pixel_compression) # create readers and update axes self.add_index_entry(index_data_entry, new_image_updates=False) # write the index to disk diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 0548b7a..903763f 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -5,6 +5,7 @@ import time import struct import warnings +import zlib from collections import OrderedDict from io import BytesIO from .file_io import NDTiffFileIO, BUILTIN_FILE_IO @@ -49,7 +50,7 @@ class SingleNDTiffWriter: - def __init__(self, directory, filename, summary_md): + def __init__(self, directory, filename, summary_md, pixel_compression = 1): self.filename = os.path.join(directory, filename) self.index_map = {} self.next_ifd_offset_location = -1 @@ -59,10 +60,15 @@ def __init__(self, directory, filename, summary_md): self.buffers = deque() self.first_ifd = True - self.start_time = None + if pixel_comepression in [1, 8]: + self.pixel_compression = pixel_compression + else: + raise ValueError("Invalid pixel compression, only 1 (no compression) and 8 (zlib) are supported") + self.start_time = None + os.makedirs(directory, exist_ok=True) - # pre-allocate the file + # pre-allocate the file file_path = os.path.join(directory, filename) with open(file_path, 'wb') as f: f.seek(MAX_FILE_SIZE - 1) @@ -132,12 +138,12 @@ def _write_null_offset_after_last_image(self): self.file.write(buffer) self.file.seek(current_pos) - def write_image(self, index_key, pixels, metadata, bit_depth='auto'): + def write_image(self, index_key, pixels, metadata, bit_depth='auto', pixel_compression = 0): """ Write an image to the file Parameters - ---------- + ---------- index_key : frozenset The key to index the image pixels : np.ndarray or bytearray @@ -152,14 +158,25 @@ def write_image(self, index_key, pixels, metadata, bit_depth='auto'): NDTiffIndexEntry The index entry for the image """ + if pixel_compression == 0: + pixel_compression == self.pixel_compression + image_height, image_width = pixels.shape rgb = pixels.ndim == 3 and pixels.shape[2] == 3 + + if rgb and pixel_compression in [8]: + warnings.warn(f"Pixel compression {pixelo_compression} is not supported for RGB images. Using no compression.") + pixel_compression = 1 + if not pixel_compression in [1,8]: + warnings.warn(f"Invalid pixel compression {pixel_compression}: only 1 (no compression) and 8 (zlib) are supported. Using 1 (no compression).") + pixel_compression = 1 + if bit_depth == 'auto': bit_depth = 8 if pixels.dtype == np.uint8 else 16 # if metadata is a dict, serialize it to a json string and make it a utf8 byte buffer if isinstance(metadata, dict): metadata = self._get_bytes_from_string(json.dumps(metadata)) - ied = self._write_ifd(index_key, pixels, metadata, rgb, image_height, image_width, bit_depth) + ied = self._write_ifd(index_key, pixels, metadata, rgb, image_height, image_width, bit_depth, pixel_compression) while self.buffers: self.file.write(self.buffers.popleft()) # make sure the file is flushed to disk @@ -168,7 +185,7 @@ def write_image(self, index_key, pixels, metadata, bit_depth='auto'): return ied - def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width, bit_depth): + def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width, bit_depth, pixel_compression): if self.file.tell() % 2 == 1: #self.file.seek(self.file.tell() + 1) # Make IFD start on word self.file.write(b'\0') # should be equivalent @@ -176,8 +193,13 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width byte_depth = 1 if isinstance(pixels, bytearray) else 2 if bit_depth == 8: byte_depth = 1 #isinstance doesen't work? - - bytes_per_image_pixels = self._bytes_per_image_pixels(pixels, rgb) + + if pixel_compression == 8: + compressed_pixels = zlib.compress(pixels) + bytes_per_image_pixels = len(compressed_pixels) + else: + bytes_per_image_pixels = self._bytes_per_image_pixels(pixels, rgb) + num_entries = 13 # 2 bytes for number of directory entries, 12 bytes per directory entry, 4 byte offset of next IFD @@ -206,7 +228,7 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, HEIGHT, 4, 1, image_height, buffer_position) buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, BITS_PER_SAMPLE, 3, 3 if rgb else 1, bits_per_sample_offset if rgb else byte_depth * 8, buffer_position) - buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, COMPRESSION, 3, 1, 1, buffer_position) + buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, COMPRESSION, 3, 1, pixel_compression, buffer_position) buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, PHOTOMETRIC_INTERPRETATION, 3, 1, 2 if rgb else 1, buffer_position) buffer_position += self._write_ifd_entry(ifd_and_small_vals_buffer, STRIP_OFFSETS, 4, 1, pixel_data_offset, @@ -239,7 +261,10 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width buffer_position += 8 self.buffers.append(ifd_and_small_vals_buffer) - self.buffers.append(self._get_pixel_buffer(pixels, rgb)) + if pixel_compression in [8]: + self.buffers.append(compressed_pixels) + else: + self.buffers.append(self._get_pixel_buffer(pixels, rgb)) self.buffers.append(metadata) self.first_ifd = False @@ -255,7 +280,7 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width }.get(bit_depth, NDTiffIndexEntry.EIGHT_BIT_RGB if rgb else None) return NDTiffIndexEntry(index_key, pixel_type, pixel_data_offset, image_width, image_height, metadata_offset, - len(metadata), self.filename.split(os.sep)[-1]) + len(metadata), self.filename.split(os.sep)[-1], pixel_compression) def _write_ifd_entry(self, buffer, tag, dtype, count, value, buffer_position): struct.pack_into(' Date: Wed, 19 Mar 2025 15:53:38 +0100 Subject: [PATCH 05/13] add multithreading with compression test --- python/ndstorage/test/test_multithreading.py | 78 ++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 python/ndstorage/test/test_multithreading.py diff --git a/python/ndstorage/test/test_multithreading.py b/python/ndstorage/test/test_multithreading.py new file mode 100644 index 0000000..5cdf993 --- /dev/null +++ b/python/ndstorage/test/test_multithreading.py @@ -0,0 +1,78 @@ +import numpy as np +import os +import shutil +import threading +import time +import sys +from collections import deque +#from ..ndtiff_file import SingleNDTiffWriter, SingleNDTiffReader +from ..ndtiff_dataset import NDTiffDataset +#from ..ndram_dataset import NDRAMDataset +import pytest + +@pytest.fixture(scope="function") +def test_data_path(tmp_path_factory): + data_path = tmp_path_factory.mktemp('writer_tests') + for f in os.listdir(data_path): + os.remove(os.path.join(data_path, f)) + yield str(data_path) + shutil.rmtree(data_path) + +# loop for threaded writing +def image_write_loop(my_deque: deque, dataset: NDTiffDataset, run_event: threading.Event): + while run_event.is_set() or len(my_deque) != 0: + try: + if my_deque: + current_time, pixels = my_deque.popleft() + axes = {'time': current_time} + dataset.put_image(axes, pixels, {'time_metadata': current_time}) + else: + time.sleep(0.001) + except IndexError: + break + +def test_write_full_dataset_multithreaded(test_data_path): + """ + Write an NDTiff dataset and read it back in, testing pixels and metadata + """ + assert sys.version_info[0] >= 3, "For test_write_full_dataset_multithreaded Python >= 3.13 is recommended" + assert sys.version_info[1] >= 13, "For test_write_full_dataset_multithreaded Python >= 3.13 is recommended" + + full_path = os.path.join(test_data_path, 'test_write_full_dataset') + dataset = NDTiffDataset(full_path, summary_metadata={}, writable=True, pixel_compression=8) + image_deque = deque() + run_event = threading.Event() + run_event.set() + + image_height = 256 + image_width = 256 + + thread = threading.Thread(target=image_write_loop, args=(image_deque, dataset, run_event)) + thread.start() + + time_counter = 0 + time_limit = 10 + + while True: + if len(image_deque) < 4: + pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_counter + image_deque.append((time_counter, pixels)) + time_counter += 1 + if time_counter >= time_limit: + break + else: + time.sleep(0.001) # if the deque is full, wait a bit + + run_event.clear() + thread.join() + dataset.finish() + + # read the file back in + dataset = NDTiffDataset(full_path) + for time_index in range(time_limit): + pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_index + axes = {'time': time_index} + read_image = dataset.read_image(**axes) + assert np.all(read_image == pixels) + assert dataset.read_metadata(**axes) == {'time_metadata': time_index} + From 5abe2181692574d8700806ffe56a1e1fcbe0cc37 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 19 Mar 2025 16:26:30 +0100 Subject: [PATCH 06/13] compression bugfix, tested --- python/ndstorage/ndtiff_dataset.py | 6 +++--- python/ndstorage/ndtiff_file.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index b1ca4aa..db2ec58 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -50,7 +50,7 @@ def __init__(self, dataset_path=None, file_io: NDTiffFileIO = BUILTIN_FILE_IO, s self.major_version = MAJOR_VERSION self.minor_version = MINOR_VERSION self._index_file = None - if pixel_compression in[1,8]: + if pixel_compression in [1,8]: self._pixel_compression = pixel_compression else: raise ValueError("Compression scheme must be 1 (No) or 8 (zlib)") @@ -180,7 +180,7 @@ def put_image(self, coordinates, image, metadata, pixel_compression = 0): if pixel_compression == 0: pixel_compression = self._pixel_compression elif not pixel_compression in [1,8]: - warnings.warn(f"Pixel compression: only 1 (no compression) and 8 (zlib) are supported. Using {self._pixel_compression}.") + warnings.warn(f"Pixel compression {pixel_compression}: only 1 (no compression) and 8 (zlib) are supported. Using {self._pixel_compression}.") pixel_compression = self._pixel_compression # add to write pending images @@ -210,7 +210,7 @@ def put_image(self, coordinates, image, metadata, pixel_compression = 0): self.current_writer = SingleNDTiffWriter(self.path, filename, self._summary_metadata, self._pixel_compression) self.file_index += 1 - index_data_entry = self.current_writer.write_image(frozenset(coordinates.items()), image, metadata, pixel_compression) + index_data_entry = self.current_writer.write_image(frozenset(coordinates.items()), image, metadata, pixel_compression = pixel_compression) # create readers and update axes self.add_index_entry(index_data_entry, new_image_updates=False) # write the index to disk diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 903763f..4ab80fc 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -60,7 +60,7 @@ def __init__(self, directory, filename, summary_md, pixel_compression = 1): self.buffers = deque() self.first_ifd = True - if pixel_comepression in [1, 8]: + if pixel_compression in [1, 8]: self.pixel_compression = pixel_compression else: raise ValueError("Invalid pixel compression, only 1 (no compression) and 8 (zlib) are supported") @@ -165,7 +165,7 @@ def write_image(self, index_key, pixels, metadata, bit_depth='auto', pixel_compr rgb = pixels.ndim == 3 and pixels.shape[2] == 3 if rgb and pixel_compression in [8]: - warnings.warn(f"Pixel compression {pixelo_compression} is not supported for RGB images. Using no compression.") + warnings.warn(f"Pixel compression {pixel_compression} is not supported for RGB images. Using no compression.") pixel_compression = 1 if not pixel_compression in [1,8]: warnings.warn(f"Invalid pixel compression {pixel_compression}: only 1 (no compression) and 8 (zlib) are supported. Using 1 (no compression).") From acaea97c5d7efff07a8e53cef6758f052c912f69 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 15:20:14 +0100 Subject: [PATCH 07/13] bugfix tiff tag bits per sample, byte_depth assignment --- python/ndstorage/ndtiff_file.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 5fcac31..2743b51 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -172,7 +172,16 @@ def _write_ifd(self, index_key, pixels, metadata, rgb, image_height, image_width if self.file.tell() % 2 == 1: self.file.seek(self.file.tell() + 1) # Make IFD start on word - byte_depth = 1 if isinstance(pixels, bytearray) else 2 + if isinstance(pixels, bytearray): + byte_depth = 1 + # if the pixel object is a numpy array, it is type of + # when using np_array.tobytes it is + # therefore taking the the bit_depth information "pixels.dtype" into account + elif bit_depth == 8: + byte_depth = 1 + else: + byte_depth = 2 + bytes_per_image_pixels = self._bytes_per_image_pixels(pixels, rgb) num_entries = 13 From e5014cb8c2a463908f27be8a101f3681cb0d14cc Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 15:31:18 +0100 Subject: [PATCH 08/13] added lock to put_image in ndtiff dataset. Heavy threaded writing did lead to double file creation and corrupted files. The lock solves this. --- python/ndstorage/ndtiff_dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index fe5a9dc..68bf2d4 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -44,6 +44,7 @@ def __init__(self, dataset_path=None, file_io: NDTiffFileIO = BUILTIN_FILE_IO, s self.file_io = file_io self._lock = threading.RLock() + self._put_image_lock = threading.Lock() if writable: self.major_version = MAJOR_VERSION self.minor_version = MINOR_VERSION @@ -166,6 +167,8 @@ def read_metadata(self, channel=None, z=None, time=None, position=None, row=None return self._do_read_metadata(axes) def put_image(self, coordinates, image, metadata): + # wait for put_image to finish before calling it again. + self._put_image_lock.acquire() if not self._writable: raise RuntimeError("Cannot write to a read-only dataset") @@ -203,6 +206,7 @@ def put_image(self, coordinates, image, metadata): self._index_file.write(index_data_entry.as_byte_buffer().getvalue()) # remove from pending images del self._write_pending_images[frozenset(coordinates.items())] + self._put_image_lock.release() def finish(self): if self.current_writer is not None: From 9df1b0a0ee60b8c8de2dcdd37d9e7b34d8a8ea1d Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 17:07:12 +0100 Subject: [PATCH 09/13] prevent duplicated reader object when writing in ndtiff dataset --- python/ndstorage/ndtiff_dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index fe5a9dc..b20ec16 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -258,7 +258,11 @@ def add_index_entry(self, data, new_image_updates=True): self.index[frozenset(image_coordinates.items())] = index_entry if index_entry.filename not in self._readers_by_filename: - new_reader = SingleNDTiffReader(os.path.join(self.path, index_entry.filename), file_io=self.file_io) + # prevent new reader object when writing: + if self._writable and self.current_writer.filename.split(os.sep)[-1] == index_entry.filename: + new_reader = self.current_writer.reader + else: + new_reader = SingleNDTiffReader(os.path.join(self.path, index_entry.filename), file_io=self.file_io) self._readers_by_filename[index_entry.filename] = new_reader # Should be the same on every file so resetting them is fine self.major_version, self.minor_version = new_reader.major_version, new_reader.minor_version From 887520d61b98fd7f4dfe2372acaa9331265dc02d Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 17:27:38 +0100 Subject: [PATCH 10/13] speed up random access reads with mmap in SingleNDTiffWriter --- python/ndstorage/ndtiff_file.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 5fcac31..963107b 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -5,6 +5,7 @@ import time import struct import warnings +import mmap from collections import OrderedDict from io import BytesIO from .file_io import NDTiffFileIO, BUILTIN_FILE_IO @@ -314,6 +315,8 @@ def __init__(self, tiff_path, file_io: NDTiffFileIO = BUILTIN_FILE_IO, summary_m self.file_io = file_io self.tiff_path = tiff_path self.file = self.file_io.open(tiff_path, "rb") + # mmap speeds up random access + self.mmap_file = mmap.mmap(self.file.fileno(), 0, prot=mmap.PROT_READ) if summary_md is None: self.summary_md, self.first_ifd_offset = self._read_header() else: @@ -323,6 +326,7 @@ def __init__(self, tiff_path, file_io: NDTiffFileIO = BUILTIN_FILE_IO, summary_m def close(self): """ """ + self.mmap_file.close() self.file.close() def _read_header(self): @@ -364,8 +368,8 @@ def _read(self, start, end): """ convert to python ints """ - self.file.seek(int(start), 0) - return self.file.read(end - start) + self.mmap_file.seek(int(start), 0) + return self.mmap_file.read(end - start) def read_metadata(self, index): return json.loads( From 29329017dc6a79a0c45317111701a550e06bc0aa Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 26 Mar 2025 15:08:05 +0100 Subject: [PATCH 11/13] bugfixes pixel compression and test --- python/ndstorage/ndtiff_dataset.py | 2 +- python/ndstorage/ndtiff_file.py | 2 +- python/ndstorage/test/test_multithreading.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index ad6c64b..edd0301 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -170,7 +170,7 @@ def read_metadata(self, channel=None, z=None, time=None, position=None, row=None return self._do_read_metadata(axes) - def put_image(self, coordinates, image, metadata): + def put_image(self, coordinates, image, metadata, pixel_compression = 0): # wait for put_image to finish before calling it again. self._put_image_lock.acquire() if not self._writable: diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 1571d54..552e6eb 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -160,7 +160,7 @@ def write_image(self, index_key, pixels, metadata, bit_depth='auto', pixel_compr The index entry for the image """ if pixel_compression == 0: - pixel_compression == self.pixel_compression + pixel_compression = self.pixel_compression image_height, image_width = pixels.shape rgb = pixels.ndim == 3 and pixels.shape[2] == 3 diff --git a/python/ndstorage/test/test_multithreading.py b/python/ndstorage/test/test_multithreading.py index 5cdf993..eb78806 100644 --- a/python/ndstorage/test/test_multithreading.py +++ b/python/ndstorage/test/test_multithreading.py @@ -19,6 +19,7 @@ def test_data_path(tmp_path_factory): shutil.rmtree(data_path) # loop for threaded writing +@pytest.mark.skip(reason="loop for threaded writing") def image_write_loop(my_deque: deque, dataset: NDTiffDataset, run_event: threading.Event): while run_event.is_set() or len(my_deque) != 0: try: From 136d57d0320586a55eb66b838e44cbeb94a376e8 Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 26 Mar 2025 16:38:16 +0100 Subject: [PATCH 12/13] renaming --- .../test/writing_multithreaded_zlib_test.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 python/ndstorage/test/writing_multithreaded_zlib_test.py diff --git a/python/ndstorage/test/writing_multithreaded_zlib_test.py b/python/ndstorage/test/writing_multithreaded_zlib_test.py new file mode 100644 index 0000000..04695f3 --- /dev/null +++ b/python/ndstorage/test/writing_multithreaded_zlib_test.py @@ -0,0 +1,80 @@ +import numpy as np +import os +import shutil +import threading +import time +import sys +from collections import deque +#from ..ndtiff_file import SingleNDTiffWriter, SingleNDTiffReader +from ..ndtiff_dataset import NDTiffDataset +#from ..ndram_dataset import NDRAMDataset +import pytest + +@pytest.fixture(scope="function") +def test_data_path(tmp_path_factory): + data_path = tmp_path_factory.mktemp('writer_tests') + for f in os.listdir(data_path): + os.remove(os.path.join(data_path, f)) + yield str(data_path) + shutil.rmtree(data_path) + +# loop for threaded writing +@pytest.mark.skip(reason="loop for threaded writing") +def image_write_loop(my_deque: deque, dataset: NDTiffDataset, run_event: threading.Event): + while run_event.is_set() or len(my_deque) != 0: + try: + if my_deque: + current_time, pixels = my_deque.popleft() + axes = {'time': current_time} + dataset.put_image(axes, pixels, {'time_metadata': current_time}) + else: + time.sleep(0.001) + except IndexError: + break + +#@pytest.mark.skipif(sys.version_info < (3, 13), reason="For test_write_multithreaded_zlib Python >= 3.13 is recommended") +def test_write_multithreaded_zlib(test_data_path): + """ + Write an NDTiff dataset and read it back in, testing pixels and metadata + """ + #assert sys.version_info[0] >= 3, "For test_write_multithreaded_zlib Python >= 3.13 is recommended" + #assert sys.version_info[1] >= 13, "For test_write_multithreaded_zlib Python >= 3.13 is recommended" + + full_path = os.path.join(test_data_path, 'test_write_full_dataset') + dataset = NDTiffDataset(full_path, summary_metadata={}, writable=True, pixel_compression=8) + image_deque = deque() + run_event = threading.Event() + run_event.set() + + image_height = 256 + image_width = 256 + + thread = threading.Thread(target=image_write_loop, args=(image_deque, dataset, run_event)) + thread.start() + + time_counter = 0 + time_limit = 10 + + while True: + if len(image_deque) < 4: + pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_counter + image_deque.append((time_counter, pixels)) + time_counter += 1 + if time_counter >= time_limit: + break + else: + time.sleep(0.001) # if the deque is full, wait a bit + + run_event.clear() + thread.join() + dataset.finish() + + # read the file back in + dataset = NDTiffDataset(full_path) + for time_index in range(time_limit): + pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_index + axes = {'time': time_index} + read_image = dataset.read_image(**axes) + assert np.all(read_image == pixels) + assert dataset.read_metadata(**axes) == {'time_metadata': time_index} + From 70af5cd380de77ebe1a186cf39f653d3dbf23b5b Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Wed, 26 Mar 2025 16:39:57 +0100 Subject: [PATCH 13/13] renaming --- python/ndstorage/test/test_multithreading.py | 79 -------------------- 1 file changed, 79 deletions(-) delete mode 100644 python/ndstorage/test/test_multithreading.py diff --git a/python/ndstorage/test/test_multithreading.py b/python/ndstorage/test/test_multithreading.py deleted file mode 100644 index eb78806..0000000 --- a/python/ndstorage/test/test_multithreading.py +++ /dev/null @@ -1,79 +0,0 @@ -import numpy as np -import os -import shutil -import threading -import time -import sys -from collections import deque -#from ..ndtiff_file import SingleNDTiffWriter, SingleNDTiffReader -from ..ndtiff_dataset import NDTiffDataset -#from ..ndram_dataset import NDRAMDataset -import pytest - -@pytest.fixture(scope="function") -def test_data_path(tmp_path_factory): - data_path = tmp_path_factory.mktemp('writer_tests') - for f in os.listdir(data_path): - os.remove(os.path.join(data_path, f)) - yield str(data_path) - shutil.rmtree(data_path) - -# loop for threaded writing -@pytest.mark.skip(reason="loop for threaded writing") -def image_write_loop(my_deque: deque, dataset: NDTiffDataset, run_event: threading.Event): - while run_event.is_set() or len(my_deque) != 0: - try: - if my_deque: - current_time, pixels = my_deque.popleft() - axes = {'time': current_time} - dataset.put_image(axes, pixels, {'time_metadata': current_time}) - else: - time.sleep(0.001) - except IndexError: - break - -def test_write_full_dataset_multithreaded(test_data_path): - """ - Write an NDTiff dataset and read it back in, testing pixels and metadata - """ - assert sys.version_info[0] >= 3, "For test_write_full_dataset_multithreaded Python >= 3.13 is recommended" - assert sys.version_info[1] >= 13, "For test_write_full_dataset_multithreaded Python >= 3.13 is recommended" - - full_path = os.path.join(test_data_path, 'test_write_full_dataset') - dataset = NDTiffDataset(full_path, summary_metadata={}, writable=True, pixel_compression=8) - image_deque = deque() - run_event = threading.Event() - run_event.set() - - image_height = 256 - image_width = 256 - - thread = threading.Thread(target=image_write_loop, args=(image_deque, dataset, run_event)) - thread.start() - - time_counter = 0 - time_limit = 10 - - while True: - if len(image_deque) < 4: - pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_counter - image_deque.append((time_counter, pixels)) - time_counter += 1 - if time_counter >= time_limit: - break - else: - time.sleep(0.001) # if the deque is full, wait a bit - - run_event.clear() - thread.join() - dataset.finish() - - # read the file back in - dataset = NDTiffDataset(full_path) - for time_index in range(time_limit): - pixels = np.ones(image_height * image_width, dtype=np.uint16).reshape((image_height, image_width)) * time_index - axes = {'time': time_index} - read_image = dataset.read_image(**axes) - assert np.all(read_image == pixels) - assert dataset.read_metadata(**axes) == {'time_metadata': time_index} -