From f86f9e8b9eca7c1d1cf7606a154c3387529238f7 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 13:05:40 -0400 Subject: [PATCH 01/18] implement yuv and 'bufferless' TextureArraY --- fastplotlib/graphics/features/_image.py | 208 ++++++++++++++++++++++-- fastplotlib/graphics/image.py | 75 ++++++--- 2 files changed, 243 insertions(+), 40 deletions(-) diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 27fd74196..69dbae9b2 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -1,11 +1,14 @@ from itertools import product from math import ceil +from typing import Literal from warnings import warn import cmap as cmap_lib import numpy as np +import wgpu import pygfx + from ._base import GraphicFeature, GraphicFeatureEvent, block_reentrance from ...utils import ( @@ -33,17 +36,50 @@ class TextureArray(GraphicFeature): }, ] - def __init__(self, data, property_name: str = "data"): + def __init__( + self, + data, + property_name: str = "data", + cpu_buffer: bool = True, + colorspace: Literal[ + "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" + ] = "srgb", + colorrange: Literal["full", "limited"] = "limited", + ): super().__init__(property_name=property_name) - data = self._fix_data(data) + if colorspace not in ("srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"): + raise ValueError( + f"`colorspace` must be one of: 'srgb', 'tex-srgb', 'physical', 'yuv420p', 'yuv444p'\n" + f"you passed: {colorspace}" + ) + + if colorrange not in ("full", "limited"): + raise ValueError( + f"`colorrange` must be one of 'full', 'limited'\n" + f"you passed: {colorrange}" + ) + + if colorspace in ("yuv420p", "yuv444p"): + # the only real use cases of yuv is video which is almost always going to be uint8 + format_ = "r8unorm" + else: + # let Texture auto-determine format + format_ = None + + data = self._check_data(data, colorspace, cpu_buffer) shared = pygfx.renderers.wgpu.get_shared() self._texture_limit_2d = shared.device.limits["max-texture-dimension-2d"] - # create a new buffer - self._value = np.zeros(data.shape, dtype=data.dtype) - self.value[:] = data[:] + if cpu_buffer: + # create a local buffer + self._value = np.zeros(data.shape, dtype=data.dtype) + self.value[:] = data[:] + usage = 0 + else: + self._value = None + usage = wgpu.TextureUsage.COPY_DST # data start indices for each Texture self._row_indices = np.arange( @@ -62,21 +98,122 @@ def __init__(self, data, property_name: str = "data"): shape=(self.row_indices.size, self.col_indices.size), dtype=object ) + if self._buffer.size > 1 and colorspace == "yuv420p": + # for now don't support yuv420p with tiling textures, too complicated + raise ValueError( + f"colorspace yuv420p is currently not supported if the image dimensions exceed the device's " + f"max-texture-dimension-2d. For now you must tile individual Images to use the yuv420p colorspace." + ) + self._iter = None + if colorspace in ("srgb", "tex-srgb", "physical"): + depth = 1 + elif colorspace == "yuv420": + depth = 2 # u and v get stored together in the 2nd layer + elif colorspace == "yuv444p": + depth = 3 # y, u, v get independent layers + + self._shape = data.shape + # iterate through each chunk of passed `data` # create a pygfx.Texture from this chunk - for _, buffer_index, data_slice in self: - texture = pygfx.Texture(self.value[data_slice], dim=2) + for _, buffer_index, slicer in self: + chunk = self.value[slicer] + + if cpu_buffer: + # texture gets the data directly + texture = pygfx.Texture( + chunk, + dim=2, + colorspace=colorspace, + colorrange=colorrange, + format=format_, + usage=usage, + ) + else: + # we only supply the size + w, h = chunk.shape[1], chunk.shape[0] + texture = pygfx.Texture( + size=(w, h, depth), + dim=2, + colorspace=colorspace, + colorrange=colorrange, + format=format_, + usage=usage, + ) + # send the initial data + if colorspace == "yuv420p": + # assume yuv data is packed, reshape and send with respective offsets + y = chunk[:h] + u = chunk[h : h + h // 4].reshape(h // 2, w // 2) + v = chunk[h + h // 4 :].reshape(h // 2, w // 2) + texture.send_data((0, 0, 0), y) + texture.send_data((0, 0, 1), u) + texture.send_data((w // 2, 0, 1), v) + else: + # all other colorspaces can be directly sent + texture.send_data((0, 0, 0), chunk) self.buffer[buffer_index] = texture + self._colorspace = colorspace + self._colorrange = colorrange + self._cpu_buffer = cpu_buffer + @property - def value(self) -> np.ndarray: + def colorspace( + self, + ) -> Literal["srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"]: + """Colorspace, read only""" + return self._colorspace + + @property + def colorrange(self) -> Literal["full", "limited"]: + """Colorspace, read only property""" + return self._colorrange + + @property + def cpu_buffer(self) -> bool: + """whether or not a cpu buffer exists for this TextureArray""" + return self._cpu_buffer + + @property + def shape(self) -> tuple[int, ...]: + """ + the shape of the represented data + if the colorspace is yuv420p then it is the shape of the _packed_ data + """ + return self._shape + + @property + def value(self) -> np.ndarray | None: + """array buffer if Texture has a cpu buffer, otherwise None""" return self._value def set_value(self, graphic, value): - self[:] = value + if self.cpu_buffer: + # if cpu_buffer is False, we directly send + if value.shape != self.shape: + raise ValueError( + f"new data shape must be the same as the original data array" + f"original data shape was: {self.shape}, data passed is of shape: {value.shape}" + ) + # send everything + if colorspace == "yuv420p": + # yuv data is packed, reshape and send with respective offsets + y = value[:h] + u = value[h : h + h // 4].reshape(h // 2, w // 2) + v = value[h + h // 4 :].reshape(h // 2, w // 2) + texture.send_data((0, 0, 0), y) + texture.send_data((0, 0, 1), u) + texture.send_data((w // 2, 0, 1), v) + else: + # all other colorspaces can be directly sent + texture.send_data((0, 0, 0), value) + else: + # set the cpu buffer, it will be marked for upload + self[:] = value @property def buffer(self) -> np.ndarray[pygfx.Texture]: @@ -98,12 +235,41 @@ def col_indices(self) -> np.ndarray: """ return self._col_indices - def _fix_data(self, data): - if data.ndim not in (2, 3): - raise ValueError( - "image data must be 2D with or without an RGB(A) dimension, i.e. " - "it must be of shape [rows, cols], [rows, cols, 3] or [rows, cols, 4]" - ) + def _check_data(self, data, colorspace, cpu_buffer): + # make sure data ndim is valid for the given colorspace + + if colorspace in ("srgb", "tex-srgb", "physical"): + if data.ndim not in (2, 3): + raise ValueError( + "if the colorspace is 'srgb', 'tex-srgb', or 'physical', " + "the image data must be 2D with or without an RGB(A) dimension, i.e. " + "it must be of shape [rows, cols], [rows, cols, 3] or [rows, cols, 4]" + ) + + if data.ndim == 3 and not cpu_buffer: + # wgpu only supports rgba, it does not support rgb + if data.shape[-1] != 4: + raise ValueError( + "if the colorspace is 'srgb', 'tex-srgb', or 'physical' and `cpu_buffer=False`" + "the image data MUST be RGBA, with shape [rows, cols, 4]. WGPU does not support " + "rgb textures. You must either supply full a RGBA array with `cpu_buffer=False` or " + "use `cpu_buffer=True` which supports RGB arrays." + ) + + elif colorspace == "yuv420p": + if data.ndim != 2: + raise ValueError( + "if the colorspace is 'yuv420p' the data array must have 2 dimensions, " + "with the `u` and `v` values packed along the bottom rows of the 2D data array" + ) + + elif colorspace == "yuv444p": + if data.ndim != 3 and data.shape[-1] != 3: + raise ValueError( + "if the colorspace is 'yuv420p' the data array must have 3 dimensions, " + "the shape should be: [rows, cols, 3], i.e. a stack of 3 2D arrays that " + "represent y, u, v." + ) if data.itemsize == 8: warn(f"casting {data.dtype} array to float32") @@ -136,18 +302,26 @@ def __next__(self) -> tuple[pygfx.Texture, tuple[int, int], tuple[slice, slice]] col_stop = min(self.value.shape[1], data_col_start + self._texture_limit_2d) # row and column slices that slice the data for this chunk from the big data array - data_slice = (slice(data_row_start, row_stop), slice(data_col_start, col_stop)) + slicer = (slice(data_row_start, row_stop), slice(data_col_start, col_stop)) # texture for this chunk texture = self.buffer[chunk_index] - return texture, chunk_index, data_slice + return texture, chunk_index, slicer def __getitem__(self, item): return self.value[item] @block_reentrance def __setitem__(self, key, value): + if not self.cpu_buffer: + raise BufferError( + f"setting slices or specific elements of texture data is only supported when `cpu_buffer=True`." + f"'unbuffered' textures only support setting the full data entirely, " + f"i.e. you must do: graphic.data = new_arr, you cannot do: graphic.data[indices] = new_arr, unless " + f"`cpu_buffer=True`" + ) + self.value[key] = value for texture in self.buffer.ravel(): diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index 8e11f4751..d2b6ee0c1 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -103,6 +103,11 @@ def __init__( cmap: str = "plasma", interpolation: str = "nearest", cmap_interpolation: str = "linear", + colorspace: Literal[ + "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" + ] = "srgb", + colorrange: Literal["full", "limited"] = "limited", + cpu_buffer: bool = True, **kwargs, ): """ @@ -139,16 +144,24 @@ def __init__( group = pygfx.Group() + self._colorspace = colorspace + self._colorrange = colorrange + if isinstance(data, TextureArray): # share buffer self._data = data else: # create new texture array to manage buffer # texture array that manages the multiple textures on the GPU that represent this image - self._data = TextureArray(data) + self._data = TextureArray( + data, + cpu_buffer=cpu_buffer, + colorspace=colorspace, + colorrange=colorrange, + ) if (vmin is None) or (vmax is None): - _vmin, _vmax = quick_min_max(self.data.value) + _vmin, _vmax = quick_min_max(data) if vmin is None: vmin = _vmin if vmax is None: @@ -161,12 +174,12 @@ def __init__( self._interpolation = ImageInterpolation(interpolation) self._cmap_interpolation = ImageCmapInterpolation(cmap_interpolation) - # set map to None for RGB images - if self._data.value.ndim == 3: + # set map to None for RGB(A) or yuv444p images + if data.ndim == 3: self._cmap = None _map = None - elif self._data.value.ndim == 2: + elif data.ndim == 2 and colorspace != "yuv420p": # use TextureMap for grayscale images self._cmap = ImageCmap(cmap) @@ -175,12 +188,6 @@ def __init__( filter=self._cmap_interpolation.value, wrap="clamp-to-edge", ) - else: - raise ValueError( - f"ImageGraphic `data` must have 2 dimensions for grayscale images, or 3 dimensions for RGB(A) images.\n" - f"You have passed a a data array with: {self._data.value.ndim} dimensions, " - f"and of shape: {self._data.value.shape}" - ) # one common material is used for every Texture chunk self._material = pygfx.ImageBasicMaterial( @@ -229,28 +236,34 @@ def data(self) -> TextureArray: Note that if the shape of the new data array does not equal the shape of current data array, a new set of GPU Textures are automatically created. - This can have performance drawbacks when you have a ver large images. + This can have performance drawbacks when you have a very large image. This is usually fine as long as you don't need to do it hundreds of times per second. """ return self._data @data.setter - def data(self, data): - if isinstance(data, np.ndarray): + def data(self, new_data): + if isinstance(new_data, np.ndarray): # check if a new buffer is required - if self._data.value.shape != data.shape: + if self._data.shape != new_data.shape: # create new TextureArray - self._data = TextureArray(data) - - # cmap based on if rgb or grayscale - if self._data.value.ndim > 2: + self._data = TextureArray( + data, + cpu_buffer=self.cpu_buffer, + colorspace=self.colorspace, + colorrange=self.colorrange, + ) + + # see if the new texture data needs a cmap + if len(self._data.shape) == 3 and self._data.colorspace != "yuv420p": + # set cmap to None since data is not grayscale self._cmap = None - - # must be None if RGB(A) self._material.map = None else: - if self.cmap is None: # have switched from RGBA -> grayscale image + if ( + self.cmap is None + ): # have switched from non-grayscale -> grayscale image # create default cmap self._cmap = ImageCmap("plasma") self._material.map = pygfx.TextureMap( @@ -274,7 +287,23 @@ def data(self, data): return - self._data[:] = data + self._data.set_value(self, new_data) + + @property + def cpu_buffer(self) -> bool: + return self.data.cpu_buffer + + @property + def colorspace( + self, + ) -> Literal["srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"]: + """colorspace, read-only property""" + return self.data.colorspace + + @propety + def colorrange(self) -> Literal["full", "limited"]: + """colorrange, read-only property""" + return self.data.colorrange @property def cmap(self) -> str | None: From 84d4df1d144989ac2026dcabe1c3bc9040176840 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 14:22:09 -0400 Subject: [PATCH 02/18] unbuffered and yuv420 works --- fastplotlib/graphics/features/_image.py | 78 +++++++++++++++++-------- fastplotlib/graphics/features/utils.py | 23 ++++++++ fastplotlib/graphics/image.py | 39 +++++++++---- 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 69dbae9b2..222ab09e0 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -11,6 +11,7 @@ from ._base import GraphicFeature, GraphicFeatureEvent, block_reentrance +from .utils import get_element_format_from_numpy_array from ...utils import ( get_cmap_texture, ) @@ -40,7 +41,7 @@ def __init__( self, data, property_name: str = "data", - cpu_buffer: bool = True, + cpu_buffer: bool = False, colorspace: Literal[ "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" ] = "srgb", @@ -60,14 +61,26 @@ def __init__( f"you passed: {colorrange}" ) + data = self._check_data(data, colorspace, cpu_buffer) + if colorspace in ("yuv420p", "yuv444p"): # the only real use cases of yuv is video which is almost always going to be uint8 format_ = "r8unorm" else: - # let Texture auto-determine format - format_ = None + # auto-determine format, adapted from pygfx.Texture + element_format = get_element_format_from_numpy_array(data) + if element_format is None: + raise ValueError( + f"Unsupported dtype/format for texture data: {data.dtype}" + ) - data = self._check_data(data, colorspace, cpu_buffer) + if data.ndim == 3: + nchannels = data.shape[-1] + else: + nchannels = 1 + format_ = (f"{nchannels}x" + element_format).lstrip("1x") + + self._shape = data.shape shared = pygfx.renderers.wgpu.get_shared() self._texture_limit_2d = shared.device.limits["max-texture-dimension-2d"] @@ -84,12 +97,12 @@ def __init__( # data start indices for each Texture self._row_indices = np.arange( 0, - ceil(self.value.shape[0] / self._texture_limit_2d) * self._texture_limit_2d, + ceil(self.shape[0] / self._texture_limit_2d) * self._texture_limit_2d, self._texture_limit_2d, ) self._col_indices = np.arange( 0, - ceil(self.value.shape[1] / self._texture_limit_2d) * self._texture_limit_2d, + ceil(self.shape[1] / self._texture_limit_2d) * self._texture_limit_2d, self._texture_limit_2d, ) @@ -109,17 +122,15 @@ def __init__( if colorspace in ("srgb", "tex-srgb", "physical"): depth = 1 - elif colorspace == "yuv420": + elif colorspace == "yuv420p": depth = 2 # u and v get stored together in the 2nd layer elif colorspace == "yuv444p": depth = 3 # y, u, v get independent layers - self._shape = data.shape - # iterate through each chunk of passed `data` # create a pygfx.Texture from this chunk for _, buffer_index, slicer in self: - chunk = self.value[slicer] + chunk = data[slicer] if cpu_buffer: # texture gets the data directly @@ -131,9 +142,17 @@ def __init__( format=format_, usage=usage, ) + print(texture.format) else: # we only supply the size - w, h = chunk.shape[1], chunk.shape[0] + if colorspace == "yuv420p": + # yuv420 data is packed, get the unpacked shape + w = chunk.shape[1] + # u, v is packed in the bottom 1/3rd of rows + h = chunk.shape[0] * 2 // 3 + else: + # otherwise just regular shape, no packing/unpacking stuff + w, h = chunk.shape[1], chunk.shape[0] texture = pygfx.Texture( size=(w, h, depth), dim=2, @@ -144,7 +163,7 @@ def __init__( ) # send the initial data if colorspace == "yuv420p": - # assume yuv data is packed, reshape and send with respective offsets + # yuv420 data is packed, unpack, reshape and send with respective offsets y = chunk[:h] u = chunk[h : h + h // 4].reshape(h // 2, w // 2) v = chunk[h + h // 4 :].reshape(h // 2, w // 2) @@ -191,26 +210,34 @@ def value(self) -> np.ndarray | None: """array buffer if Texture has a cpu buffer, otherwise None""" return self._value - def set_value(self, graphic, value): - if self.cpu_buffer: - # if cpu_buffer is False, we directly send + def set_value(self, graphic, value: np.ndarray): + if not self.cpu_buffer: + # if cpu_buffer is False, we directly send data to the GPU if value.shape != self.shape: raise ValueError( f"new data shape must be the same as the original data array" f"original data shape was: {self.shape}, data passed is of shape: {value.shape}" ) # send everything - if colorspace == "yuv420p": - # yuv data is packed, reshape and send with respective offsets + if self.colorspace == "yuv420p": + # yuv data is packed. unpack, reshape and send with respective offsets + w = value.shape[1] + # u, v is packed in the bottom 1/3rd of rows + h = value.shape[0] * 2 // 3 + y = value[:h] u = value[h : h + h // 4].reshape(h // 2, w // 2) v = value[h + h // 4 :].reshape(h // 2, w // 2) - texture.send_data((0, 0, 0), y) - texture.send_data((0, 0, 1), u) - texture.send_data((w // 2, 0, 1), v) + + self._buffer[0, 0].send_data((0, 0, 0), y) + self._buffer[0, 0].send_data((0, 0, 1), u) + self._buffer[0, 0].send_data((w // 2, 0, 1), v) else: # all other colorspaces can be directly sent - texture.send_data((0, 0, 0), value) + for texture, buffer_index, slicer in self: + chunk = value[slicer] + texture.send_data((0, 0, 0), chunk) + else: # set the cpu buffer, it will be marked for upload self[:] = value @@ -264,7 +291,7 @@ def _check_data(self, data, colorspace, cpu_buffer): ) elif colorspace == "yuv444p": - if data.ndim != 3 and data.shape[-1] != 3: + if data.ndim != 3 or data.shape[-1] != 3: raise ValueError( "if the colorspace is 'yuv420p' the data array must have 3 dimensions, " "the shape should be: [rows, cols, 3], i.e. a stack of 3 2D arrays that " @@ -298,8 +325,8 @@ def __next__(self) -> tuple[pygfx.Texture, tuple[int, int], tuple[slice, slice]] chunk_index = (chunk_row, chunk_col) # stop indices of big data array for this chunk - row_stop = min(self.value.shape[0], data_row_start + self._texture_limit_2d) - col_stop = min(self.value.shape[1], data_col_start + self._texture_limit_2d) + row_stop = min(self.shape[0], data_row_start + self._texture_limit_2d) + col_stop = min(self.shape[1], data_col_start + self._texture_limit_2d) # row and column slices that slice the data for this chunk from the big data array slicer = (slice(data_row_start, row_stop), slice(data_col_start, col_stop)) @@ -310,6 +337,9 @@ def __next__(self) -> tuple[pygfx.Texture, tuple[int, int], tuple[slice, slice]] return texture, chunk_index, slicer def __getitem__(self, item): + if not self.cpu_buffer: + return None + return self.value[item] @block_reentrance diff --git a/fastplotlib/graphics/features/utils.py b/fastplotlib/graphics/features/utils.py index aa4022052..ef67297ce 100644 --- a/fastplotlib/graphics/features/utils.py +++ b/fastplotlib/graphics/features/utils.py @@ -77,3 +77,26 @@ def parse_colors( data = make_pygfx_colors(colors, n_colors) return to_gpu_supported_dtype(data) + + +def get_element_format_from_numpy_array(array): + """Get the per-element format specifier from a numpy array. + Returns None if the format appears to be a structured array. + Raises an error if GPU-incompatible dtypes are used (64 bit). + """ + + # Uniform buffers are scalars with a structured dtype. + # But can also create storage buffers with complex formats. + if array.dtype.kind not in "iuf": + return None + + # GPUs generally don't support 64-bit buffers or textures. + # Note: the Python docs say that l and L are 32 bit, but converting + # a int64 numpy array to a memoryview gives a format of 'l' instead + # of 'q' on some systems/configs? So we need to check the itemsize. + if array.itemsize == 8: + raise ValueError( + f"A dtype of {array.dtype.name} is not supported for buffers, use a 32-bit variant instead." + ) + + return array.dtype.str.lstrip("<>=|") \ No newline at end of file diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index d2b6ee0c1..42bd87dd4 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -3,6 +3,7 @@ import numpy as np import pygfx +from pygfx import Texture from ..utils import quick_min_max from ._base import Graphic @@ -107,7 +108,7 @@ def __init__( "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" ] = "srgb", colorrange: Literal["full", "limited"] = "limited", - cpu_buffer: bool = True, + cpu_buffer: bool = False, **kwargs, ): """ @@ -174,12 +175,11 @@ def __init__( self._interpolation = ImageInterpolation(interpolation) self._cmap_interpolation = ImageCmapInterpolation(cmap_interpolation) - # set map to None for RGB(A) or yuv444p images - if data.ndim == 3: - self._cmap = None - _map = None + # cmap only used for grayscale images + self._cmap = None + _map = None - elif data.ndim == 2 and colorspace != "yuv420p": + if data.ndim == 2 and colorspace != "yuv420p": # use TextureMap for grayscale images self._cmap = ImageCmap(cmap) @@ -249,7 +249,7 @@ def data(self, new_data): if self._data.shape != new_data.shape: # create new TextureArray self._data = TextureArray( - data, + new_data, cpu_buffer=self.cpu_buffer, colorspace=self.colorspace, colorrange=self.colorrange, @@ -300,7 +300,7 @@ def colorspace( """colorspace, read-only property""" return self.data.colorspace - @propety + @property def colorrange(self) -> Literal["full", "limited"]: """colorrange, read-only property""" return self.data.colorrange @@ -317,8 +317,9 @@ def cmap(self) -> str | None: @cmap.setter def cmap(self, name: str): - if self.data.value.ndim > 2: - raise AttributeError("RGB(A) images do not have a colormap property") + if len(self.data.shape) > 2: + raise AttributeError("cmap is only supported for grayscale images") + self._cmap.set_value(self, name) @property @@ -361,6 +362,8 @@ def reset_vmin_vmax(self): """ Reset the vmin, vmax by estimating it from the data by subsampling. """ + if not self.cpu_buffer: + return vmin, vmax = quick_min_max(self._data.value) self.vmin = vmin @@ -569,6 +572,22 @@ def add_polygon_selector( return selector def format_pick_info(self, pick_info: dict) -> str: + if not self.cpu_buffer: + if self.data.colorspace != "yuv420p" and len(self.data.shape) == 2: + # inverse map from rgb pixel value to grayscale value using the colormap + # we can only perform a guess + lut = self._material.map.texture.data + rgb = pick_info["rgba"][:3] + closest = np.argmin(np.linalg.norm(lut[:, :3] - rgb, axis=1)) + scalar = closest / (lut.shape[0] - 1) + val = self.vmin + scalar * (self.vmax - self.vmin) + return f"{val:.4g}" + else: + # rgba vals + rgba_val = pick_info["rgba"] + info = "\n".join(f"{channel}: {val: .4g}" for channel, val in zip("rgba", rgba_val)) + return info + col, row = pick_info["index"] if self.data.value.ndim == 2: val = self.data[row, col] From 3a9b13dd67b80e5967d1b3e0d0be90057f64cd47 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 14:24:00 -0400 Subject: [PATCH 03/18] warning on tooltip --- fastplotlib/graphics/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index 42bd87dd4..a6bf86f93 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -581,7 +581,7 @@ def format_pick_info(self, pick_info: dict) -> str: closest = np.argmin(np.linalg.norm(lut[:, :3] - rgb, axis=1)) scalar = closest / (lut.shape[0] - 1) val = self.vmin + scalar * (self.vmax - self.vmin) - return f"{val:.4g}" + return f"{val:.4g}\n!!estimate!!, cpu_buffer=False" else: # rgba vals rgba_val = pick_info["rgba"] From b28418bf3ecb1ad10fa88d4f9c5c9628957d1f01 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 14:27:05 -0400 Subject: [PATCH 04/18] NDImage always uses unbuffered, support colorspaces in NDIMage --- fastplotlib/widgets/nd_widget/_base.py | 4 +++- fastplotlib/widgets/nd_widget/_nd_image.py | 24 ++++++++++++++++--- fastplotlib/widgets/nd_widget/_ndw_subplot.py | 10 ++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/fastplotlib/widgets/nd_widget/_base.py b/fastplotlib/widgets/nd_widget/_base.py index 5c2747d2b..724f5fdd6 100644 --- a/fastplotlib/widgets/nd_widget/_base.py +++ b/fastplotlib/widgets/nd_widget/_base.py @@ -501,7 +501,9 @@ def get_window_output(self, indices: dict[str, Any]) -> AwaitedArray: windowed_slice = windowed_slice.squeeze(axis=slider_dims_int) if windowed_slice.ndim != len(self.spatial_dims): - raise ValueError + raise ValueError( + f"windowed_slice.ndim != len(self.spatial_dims): {windowed_slice.ndim} != {len(self.spatial_dims)}" + ) # transpose to spatial dims spatial_dims_int = tuple( diff --git a/fastplotlib/widgets/nd_widget/_nd_image.py b/fastplotlib/widgets/nd_widget/_nd_image.py index 9fa39606d..0796c838b 100644 --- a/fastplotlib/widgets/nd_widget/_nd_image.py +++ b/fastplotlib/widgets/nd_widget/_nd_image.py @@ -1,5 +1,5 @@ from collections.abc import Sequence, Generator -from typing import Callable, Any +from typing import Callable, Any, Literal import numpy as np from numpy.typing import ArrayLike @@ -8,7 +8,13 @@ from ...utils import subsample_array, ARRAY_LIKE_ATTRS, ArrayProtocol from ...graphics import ImageGraphic, ImageVolumeGraphic from ...tools import HistogramLUTTool -from ._base import NDProcessor, NDGraphic, WindowFuncCallable, block_reentrance, AwaitedArray +from ._base import ( + NDProcessor, + NDGraphic, + WindowFuncCallable, + block_reentrance, + AwaitedArray, +) from ._index import ReferenceIndex from ._async import start_coroutine @@ -284,6 +290,10 @@ def __init__( spatial_func: Callable[[ArrayLike], ArrayLike] = None, compute_histogram: bool = True, slider_dim_transforms=None, + colorspace: Literal[ + "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" + ] = "srgb", + colorrange: Literal["full", "limited"] = "full", name: str = None, ): """ @@ -371,6 +381,9 @@ def __init__( slider_dim_transforms=slider_dim_transforms, ) + self._colorspace = colorspace + self._colorrange = colorrange + self._graphic: ImageGraphic | None = None self._histogram_widget: HistogramLUTTool | None = None @@ -414,7 +427,12 @@ def _create_graphic(self): data_slice = yield from self._get_data_slice(self.indices) # create the new graphic - new_graphic = cls(data_slice) + new_graphic = cls( + data_slice, + cpu_buffer=False, # faster, we usually don't need a cpu buffer for NDWidget use cases + colorspace=self._colorspace, + colorrange=self._colorrange, + ) old_graphic = self._graphic # check if we are replacing a graphic diff --git a/fastplotlib/widgets/nd_widget/_ndw_subplot.py b/fastplotlib/widgets/nd_widget/_ndw_subplot.py index 6666b3fc1..d9aa1fecf 100644 --- a/fastplotlib/widgets/nd_widget/_ndw_subplot.py +++ b/fastplotlib/widgets/nd_widget/_ndw_subplot.py @@ -20,6 +20,7 @@ class NDWSubplot: Note: ``NDWSubplot`` is not meant to be constructed directly, it only exists as part of an ``NDWidget`` """ + def __init__(self, ndw, subplot: Subplot): self.ndw = ndw self._subplot = subplot @@ -57,8 +58,12 @@ def add_nd_image( compute_histogram: bool = True, slider_dim_transforms=None, name: str = None, + **kwargs, ): - nd = NDImage(self.ndw.indices, self._subplot, data=data, + nd = NDImage( + self.ndw.indices, + self._subplot, + data=data, dims=dims, spatial_dims=spatial_dims, rgb_dim=rgb_dim, @@ -68,7 +73,8 @@ def add_nd_image( compute_histogram=compute_histogram, slider_dim_transforms=slider_dim_transforms, name=name, - ) + **kwargs, + ) self._nd_graphics.append(nd) return nd From 37741d095603dc90b68616c597f0c14f690700ec Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 18:38:15 -0400 Subject: [PATCH 05/18] update docstrings --- fastplotlib/graphics/features/_image.py | 2 +- fastplotlib/graphics/image.py | 80 ++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 222ab09e0..7b9ffcb87 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -41,7 +41,7 @@ def __init__( self, data, property_name: str = "data", - cpu_buffer: bool = False, + cpu_buffer: bool = True, colorspace: Literal[ "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" ] = "srgb", diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index a6bf86f93..9186f4f48 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -107,8 +107,8 @@ def __init__( colorspace: Literal[ "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" ] = "srgb", - colorrange: Literal["full", "limited"] = "limited", - cpu_buffer: bool = False, + colorrange: Literal["full", "limited"] = "full", + cpu_buffer: bool = True, **kwargs, ): """ @@ -118,6 +118,7 @@ def __init__( ---------- data: array-like array-like, usually numpy.ndarray, must support ``memoryview()`` + # TODO: update this, and also allow tuple/list of arrays for yuv420p | shape must be ``[n_rows, n_cols]``, ``[n_rows, n_cols, 3]`` for RGB or ``[n_rows, n_cols, 4]`` for RGBA vmin: float, optional @@ -136,6 +137,77 @@ def __init__( cmap_interpolation: str, optional, default "linear" colormap interpolation method, one of "nearest" or "linear" + colorspace: one of "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p", default "srgb" + colorspace in which to interpret the provided data. + + * "srgb": the data represents intensity, rgb, or rgba pixels in the sRGB space. + sRGB is a standard color space designed for consistent representation of colors + across devices like monitors. Most images store colors in this space. + The shader convers sRGB colors to physical in the shader before doing color computations. + + * "tex-srgb": the underlying texture will be of an sRGB format. This means the data + is automatically converted to sRGB when it is sampled. This results in better glTF + compliance (because interpolation in the sampling happens in linear space). + Note that sampling *always* results in the sRGB values, also when not interpreted as color. + Only supported for rgb and rgba data. + + * "physical": the colors are (already) in the physical / linear space, where lighting + calculations can be applied. Shader code that interprets the data as color will use it as-is. + + * "yuv420p": A common video format. The data is represented as 3 planes (y, u, and v). + The y represents intensity, and is at full resolution. The u and v planes are a + quarter of the size. data must be a 2D array which packs y, u and v: + + ====== + | y | + | . | + | . | + | . | + ------ + | u | + ------ + | v | + ====== + + This is the same as the packed array structure that pyav provides when reading video in as yuv420p. + + If the data represents an image with width and height (w, h), then the packed data array must be of + shape: [w, h * 3 // 2]. + + # TODO: You can also provide a tuple of arrays to data: (y, u, v) + + For more info see: https://docs.pygfx.org/stable/_gallery/feature_demo/video_yuv.html + and https://github.com/pygfx/pygfx/pull/873 + + + * "yuv444p": A lesser common video format. The data is represented as 3 planes + (y, u, and v) similar to yuv420p however the u and v planes are stored + at full resolution. + + colorrange: Literal["full", "limited"] = "limited", + Relevant for yuv colorspaces. Most videos use "limited". + + * "limited": The luma plane (Y) is limited to the range of 16-235 for 8 bits. + The chroma planes (U and V) are limited to the range of 16-240 for 8 bits + * "full": The luma plane and chroma plane use the full range of the storage format. + + See the following links from the FFMPEG documentation for more details: + https://trac.ffmpeg.org/wiki/colorspace + https://ffmpeg.org/doxygen/7.0/pixfmt_8h_source.html#l00609 + + cpu_buffer: bool, default True + If ``True``, maintains a buffer of system RAM that is sychronized with a corresponding storage buffer + on the GPU. + If ``False``, setting the graphic data will send the new data directly to the GPU, we also + call this "bufferless". This is much faster but lacks the following features: + * you must update the entire data array, i.e. you can perform ``image.data = new_data``, and you + cannot perform partial updates such as ``image.data[indices] = ``. + * RGB arrays of shape [rows, cols, 3] are not supported since wgpu does not have RGB textures, + use RGBA or use `cpu_buffer=True` if you really need RGB instead of RGBA. + * tooltip values for grayscale data are estimated using an inverse transforms on the colormap LUT. + The tooltip values may or may not be accurate for a given colormap and vmin, vmax. If you require + precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. + kwargs: additional keyword arguments passed to :class:`.Graphic` @@ -585,7 +657,9 @@ def format_pick_info(self, pick_info: dict) -> str: else: # rgba vals rgba_val = pick_info["rgba"] - info = "\n".join(f"{channel}: {val: .4g}" for channel, val in zip("rgba", rgba_val)) + info = "\n".join( + f"{channel}: {val: .4g}" for channel, val in zip("rgba", rgba_val) + ) return info col, row = pick_info["index"] From bd8a0fbfc166b680d13aea23fbab4b51d539a6de Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Mon, 13 Apr 2026 19:09:03 -0400 Subject: [PATCH 06/18] by default disable AA and set pixel_scale=1.0 for performance --- fastplotlib/layouts/_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fastplotlib/layouts/_utils.py b/fastplotlib/layouts/_utils.py index 49120c71a..453b1ce11 100644 --- a/fastplotlib/layouts/_utils.py +++ b/fastplotlib/layouts/_utils.py @@ -4,7 +4,7 @@ import numpy as np import pygfx -from pygfx import WgpuRenderer, Texture, Renderer +from pygfx import WgpuRenderer, Texture from ..utils.gui import BaseRenderCanvas, RenderCanvas @@ -22,7 +22,7 @@ def make_canvas_and_renderer( canvas: str | BaseRenderCanvas | Texture | None, - renderer: Renderer | None, + renderer: WgpuRenderer | None, canvas_kwargs: dict, ): """ @@ -45,9 +45,13 @@ def make_canvas_and_renderer( if renderer is None: renderer = WgpuRenderer(canvas) - elif not isinstance(renderer, Renderer): + + # disable AA and set pixel_scale = 1.0 for performance + renderer.ppaa = "none" + renderer.pixel_scale = 1.0 + elif not isinstance(renderer, WgpuRenderer): raise TypeError( - f"renderer option must be a pygfx.Renderer instance such as pygfx.WgpuRenderer" + f"renderer option must be a pygfx.WgpuRenderer instance" ) return canvas, renderer From a0f2aa17dc1b4514eae1373bc1350793b7e51fd0 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Wed, 15 Apr 2026 00:48:32 -0400 Subject: [PATCH 07/18] unpacked yuv support --- fastplotlib/graphics/features/_image.py | 97 ++++++++++++++++------ fastplotlib/graphics/image.py | 4 + fastplotlib/widgets/nd_widget/__init__.py | 1 + fastplotlib/widgets/nd_widget/_nd_image.py | 3 +- fastplotlib/widgets/nd_widget/_video.py | 27 ++++++ 5 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 fastplotlib/widgets/nd_widget/_video.py diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 7b9ffcb87..427d4a3ae 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -66,6 +66,11 @@ def __init__( if colorspace in ("yuv420p", "yuv444p"): # the only real use cases of yuv is video which is almost always going to be uint8 format_ = "r8unorm" + if isinstance(data, (tuple, list)): + # yuv each provided separately + self._shape = data[0].shape + else: + self._shape = data.shape[0] * 2 // 3, data.shape[0][1] else: # auto-determine format, adapted from pygfx.Texture element_format = get_element_format_from_numpy_array(data) @@ -80,13 +85,23 @@ def __init__( nchannels = 1 format_ = (f"{nchannels}x" + element_format).lstrip("1x") - self._shape = data.shape + self._shape = data.shape shared = pygfx.renderers.wgpu.get_shared() self._texture_limit_2d = shared.device.limits["max-texture-dimension-2d"] if cpu_buffer: # create a local buffer + if colorspace in ("yuv420p", "yuv444p"): + raise ValueError("yuv is currently not supported with `cpu_buffer=False") + # TODO: support yuv with local cpu buffers + # if colorspace in ("yuv420p", "yuv444p") and isinstance(data, (tuple, list)): + # self._value = ( + # np.zeros(data[0].shape, dtype=data[0].dtype), + # np.zeros(data[1].shape, dtype=data[1].dtype), + # np.zeros(data[2].shape, dtype=data[2].dtype), + # ) + # else: self._value = np.zeros(data.shape, dtype=data.dtype) self.value[:] = data[:] usage = 0 @@ -130,7 +145,11 @@ def __init__( # iterate through each chunk of passed `data` # create a pygfx.Texture from this chunk for _, buffer_index, slicer in self: - chunk = data[slicer] + if isinstance(data, (tuple, list)): + # We only support upto max 2D texture limit for YUV anyways so don't need to slice + chunk = data + else: + chunk = data[slicer] if cpu_buffer: # texture gets the data directly @@ -142,17 +161,21 @@ def __init__( format=format_, usage=usage, ) - print(texture.format) else: # we only supply the size if colorspace == "yuv420p": - # yuv420 data is packed, get the unpacked shape - w = chunk.shape[1] - # u, v is packed in the bottom 1/3rd of rows - h = chunk.shape[0] * 2 // 3 + if isinstance(chunk, (tuple, list)): + # not packed, rows, cols -> width, height + w, h = chunk[0].shape[1], chunk[0].shape[0] + else: + # yuv420 data is packed, get the unpacked shape + w = chunk.shape[1] + # u, v is packed in the bottom 1/3rd of rows + h = chunk.shape[0] * 2 // 3 else: # otherwise just regular shape, no packing/unpacking stuff w, h = chunk.shape[1], chunk.shape[0] + texture = pygfx.Texture( size=(w, h, depth), dim=2, @@ -161,12 +184,17 @@ def __init__( format=format_, usage=usage, ) + # send the initial data if colorspace == "yuv420p": - # yuv420 data is packed, unpack, reshape and send with respective offsets - y = chunk[:h] - u = chunk[h : h + h // 4].reshape(h // 2, w // 2) - v = chunk[h + h // 4 :].reshape(h // 2, w // 2) + if isinstance(chunk, (tuple, list)): + y, u, v = chunk + else: + # yuv420 data is packed, unpack, reshape and send with respective offsets + y = chunk[:h] + u = chunk[h : h + h // 4].reshape(h // 2, w // 2) + v = chunk[h + h // 4 :].reshape(h // 2, w // 2) + texture.send_data((0, 0, 0), y) texture.send_data((0, 0, 1), u) texture.send_data((w // 2, 0, 1), v) @@ -212,26 +240,33 @@ def value(self) -> np.ndarray | None: def set_value(self, graphic, value: np.ndarray): if not self.cpu_buffer: - # if cpu_buffer is False, we directly send data to the GPU - if value.shape != self.shape: - raise ValueError( - f"new data shape must be the same as the original data array" - f"original data shape was: {self.shape}, data passed is of shape: {value.shape}" - ) + if isinstance(value, np.ndarray): + # if cpu_buffer is False, we directly send data to the GPU + if value.shape != self.shape: + raise ValueError( + f"new data shape must be the same as the original data array" + f"original data shape was: {self.shape}, data passed is of shape: {value.shape}" + ) # send everything if self.colorspace == "yuv420p": - # yuv data is packed. unpack, reshape and send with respective offsets - w = value.shape[1] - # u, v is packed in the bottom 1/3rd of rows - h = value.shape[0] * 2 // 3 + if isinstance(value, (tuple, list)): + if not len(value) == 3: + raise ValueError + y, u, v = value + else: + # yuv data is packed. unpack, reshape and send with respective offsets + w = value.shape[1] + # u, v is packed in the bottom 1/3rd of rows + h = value.shape[0] * 2 // 3 - y = value[:h] - u = value[h : h + h // 4].reshape(h // 2, w // 2) - v = value[h + h // 4 :].reshape(h // 2, w // 2) + y = value[:h] + u = value[h : h + h // 4].reshape(h // 2, w // 2) + v = value[h + h // 4 :].reshape(h // 2, w // 2) self._buffer[0, 0].send_data((0, 0, 0), y) self._buffer[0, 0].send_data((0, 0, 1), u) - self._buffer[0, 0].send_data((w // 2, 0, 1), v) + # width is y.shape[1] + self._buffer[0, 0].send_data((y.shape[1] // 2, 0, 1), v) else: # all other colorspaces can be directly sent for texture, buffer_index, slicer in self: @@ -284,7 +319,17 @@ def _check_data(self, data, colorspace, cpu_buffer): ) elif colorspace == "yuv420p": - if data.ndim != 2: + if isinstance(data, (tuple, list)): + if not len(data) == 3: + raise ValueError( + f"if `colorspace=yuv420p`, must provide a tuple/list of arrays representing y, u, v components" + f"(luma, and chroma channels), or a single arrays with packed yuv components. You passed: " + f"{data}" + ) + return data + # TODO: more validation for YUV expected dims + + elif data.ndim != 2: raise ValueError( "if the colorspace is 'yuv420p' the data array must have 2 dimensions, " "with the `u` and `v` values packed along the bottom rows of the 2D data array" diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index 9186f4f48..fe2e75082 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -233,6 +233,10 @@ def __init__( colorrange=colorrange, ) + if isinstance(data, (tuple, list)): + # unpacked yuv + data = data[0] + if (vmin is None) or (vmax is None): _vmin, _vmax = quick_min_max(data) if vmin is None: diff --git a/fastplotlib/widgets/nd_widget/__init__.py b/fastplotlib/widgets/nd_widget/__init__.py index 65f448b54..f66b52ed4 100644 --- a/fastplotlib/widgets/nd_widget/__init__.py +++ b/fastplotlib/widgets/nd_widget/__init__.py @@ -5,6 +5,7 @@ from ._base import NDProcessor, NDGraphic from ._nd_positions import NDPositions, NDPositionsProcessor, ndp_extras from ._nd_image import NDImageProcessor, NDImage + from ._video import VideoProcessor from ._ndwidget import NDWidget else: diff --git a/fastplotlib/widgets/nd_widget/_nd_image.py b/fastplotlib/widgets/nd_widget/_nd_image.py index 0796c838b..5c8942363 100644 --- a/fastplotlib/widgets/nd_widget/_nd_image.py +++ b/fastplotlib/widgets/nd_widget/_nd_image.py @@ -290,6 +290,7 @@ def __init__( spatial_func: Callable[[ArrayLike], ArrayLike] = None, compute_histogram: bool = True, slider_dim_transforms=None, + processor_type: type[NDImageProcessor] = NDImageProcessor, colorspace: Literal[ "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" ] = "srgb", @@ -369,7 +370,7 @@ def __init__( self._ref_index = ref_index - self._processor = NDImageProcessor( + self._processor = processor_type( data, dims=dims, spatial_dims=spatial_dims, diff --git a/fastplotlib/widgets/nd_widget/_video.py b/fastplotlib/widgets/nd_widget/_video.py new file mode 100644 index 000000000..f1dab6f9e --- /dev/null +++ b/fastplotlib/widgets/nd_widget/_video.py @@ -0,0 +1,27 @@ +from ._nd_image import NDImageProcessor, NDImage +from typing import Callable, Any, Literal + +import numpy as np + + +class VideoProcessor(NDImageProcessor): + def get_window_output(self, indices: dict[str, Any]): + """ + Applies any window functions and returns squeezed sliced array transposed in the order of the given spatial dims + + Parameters + ---------- + indices + + Returns + ------- + + """ + # windowed slice if user set any window funcs + windowed_slice = yield from self._get_raw_data_slice(indices) + + if isinstance(windowed_slice, (tuple, list)): + return tuple(a.squeeze() for a in windowed_slice) + + # convert to numpy array + return np.asarray(windowed_slice) From 0a90134c29d927c2caaca96cee4554f91e808c7f Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 18:14:17 -0400 Subject: [PATCH 08/18] independent graphics and texture features for rgb and yuv --- fastplotlib/graphics/features/__init__.py | 4 + fastplotlib/graphics/features/_image.py | 349 +++++++++++----------- fastplotlib/graphics/image.py | 312 +++++++++++-------- fastplotlib/utils/enums.py | 18 +- 4 files changed, 391 insertions(+), 292 deletions(-) diff --git a/fastplotlib/graphics/features/__init__.py b/fastplotlib/graphics/features/__init__.py index 7f7410cf7..a04b1c991 100644 --- a/fastplotlib/graphics/features/__init__.py +++ b/fastplotlib/graphics/features/__init__.py @@ -27,6 +27,8 @@ ) from ._image import ( TextureArray, + TextureYUV, + TupleYUV, ImageCmap, ImageVmin, ImageVmax, @@ -93,6 +95,8 @@ "VertexPointSizes", "UniformSize", "TextureArray", + "TextureYUV", + "TupleYUV", "ImageCmap", "ImageVmin", "ImageVmax", diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 427d4a3ae..7ae0c2a75 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -1,10 +1,11 @@ from itertools import product from math import ceil -from typing import Literal +from typing import Literal, TypeAlias from warnings import warn import cmap as cmap_lib import numpy as np +from numpy.typing import NDArray import wgpu import pygfx @@ -12,9 +13,9 @@ from ._base import GraphicFeature, GraphicFeatureEvent, block_reentrance from .utils import get_element_format_from_numpy_array -from ...utils import ( - get_cmap_texture, -) +from ...utils import get_cmap_texture, ColorspacesRGB, ColorspacesYUV, ColorRange + +TupleYUV: TypeAlias = tuple[NDArray[np.uint8], NDArray[np.uint8], NDArray[np.uint8]] class TextureArray(GraphicFeature): @@ -42,36 +43,25 @@ def __init__( data, property_name: str = "data", cpu_buffer: bool = True, - colorspace: Literal[ - "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" - ] = "srgb", - colorrange: Literal["full", "limited"] = "limited", + colorspace: ColorspacesRGB = ColorspacesRGB.srgb, ): super().__init__(property_name=property_name) - if colorspace not in ("srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"): - raise ValueError( - f"`colorspace` must be one of: 'srgb', 'tex-srgb', 'physical', 'yuv420p', 'yuv444p'\n" - f"you passed: {colorspace}" - ) + self._colorspace = ColorspacesRGB(colorspace) + data = self._check_data(data, colorspace, cpu_buffer) - if colorrange not in ("full", "limited"): - raise ValueError( - f"`colorrange` must be one of 'full', 'limited'\n" - f"you passed: {colorrange}" - ) + self._shape = data.shape - data = self._check_data(data, colorspace, cpu_buffer) + shared = pygfx.renderers.wgpu.get_shared() + self._texture_limit_2d = shared.device.limits["max-texture-dimension-2d"] - if colorspace in ("yuv420p", "yuv444p"): - # the only real use cases of yuv is video which is almost always going to be uint8 - format_ = "r8unorm" - if isinstance(data, (tuple, list)): - # yuv each provided separately - self._shape = data[0].shape - else: - self._shape = data.shape[0] * 2 // 3, data.shape[0][1] + if cpu_buffer: + # create a local buffer + self._value = np.zeros(data.shape, dtype=data.dtype) + self.value[:] = data[:] else: + self._value = None + usage = wgpu.TextureUsage.COPY_DST # auto-determine format, adapted from pygfx.Texture element_format = get_element_format_from_numpy_array(data) if element_format is None: @@ -87,27 +77,6 @@ def __init__( self._shape = data.shape - shared = pygfx.renderers.wgpu.get_shared() - self._texture_limit_2d = shared.device.limits["max-texture-dimension-2d"] - - if cpu_buffer: - # create a local buffer - if colorspace in ("yuv420p", "yuv444p"): - raise ValueError("yuv is currently not supported with `cpu_buffer=False") - # TODO: support yuv with local cpu buffers - # if colorspace in ("yuv420p", "yuv444p") and isinstance(data, (tuple, list)): - # self._value = ( - # np.zeros(data[0].shape, dtype=data[0].dtype), - # np.zeros(data[1].shape, dtype=data[1].dtype), - # np.zeros(data[2].shape, dtype=data[2].dtype), - # ) - # else: - self._value = np.zeros(data.shape, dtype=data.dtype) - self.value[:] = data[:] - usage = 0 - else: - self._value = None - usage = wgpu.TextureUsage.COPY_DST # data start indices for each Texture self._row_indices = np.arange( @@ -122,34 +91,16 @@ def __init__( ) # buffer will be an array of textures - self._buffer: np.ndarray[pygfx.Texture] = np.empty( + self._buffer: NDArray[pygfx.Texture] = np.empty( shape=(self.row_indices.size, self.col_indices.size), dtype=object ) - if self._buffer.size > 1 and colorspace == "yuv420p": - # for now don't support yuv420p with tiling textures, too complicated - raise ValueError( - f"colorspace yuv420p is currently not supported if the image dimensions exceed the device's " - f"max-texture-dimension-2d. For now you must tile individual Images to use the yuv420p colorspace." - ) - self._iter = None - if colorspace in ("srgb", "tex-srgb", "physical"): - depth = 1 - elif colorspace == "yuv420p": - depth = 2 # u and v get stored together in the 2nd layer - elif colorspace == "yuv444p": - depth = 3 # y, u, v get independent layers - # iterate through each chunk of passed `data` # create a pygfx.Texture from this chunk for _, buffer_index, slicer in self: - if isinstance(data, (tuple, list)): - # We only support upto max 2D texture limit for YUV anyways so don't need to slice - chunk = data - else: - chunk = data[slicer] + chunk = data[slicer] if cpu_buffer: # texture gets the data directly @@ -157,79 +108,43 @@ def __init__( chunk, dim=2, colorspace=colorspace, - colorrange=colorrange, - format=format_, - usage=usage, ) else: # we only supply the size - if colorspace == "yuv420p": - if isinstance(chunk, (tuple, list)): - # not packed, rows, cols -> width, height - w, h = chunk[0].shape[1], chunk[0].shape[0] - else: - # yuv420 data is packed, get the unpacked shape - w = chunk.shape[1] - # u, v is packed in the bottom 1/3rd of rows - h = chunk.shape[0] * 2 // 3 - else: - # otherwise just regular shape, no packing/unpacking stuff - w, h = chunk.shape[1], chunk.shape[0] + w, h = chunk.shape[1], chunk.shape[0] texture = pygfx.Texture( - size=(w, h, depth), + size=(w, h, 1), dim=2, colorspace=colorspace, - colorrange=colorrange, format=format_, usage=usage, ) # send the initial data - if colorspace == "yuv420p": - if isinstance(chunk, (tuple, list)): - y, u, v = chunk - else: - # yuv420 data is packed, unpack, reshape and send with respective offsets - y = chunk[:h] - u = chunk[h : h + h // 4].reshape(h // 2, w // 2) - v = chunk[h + h // 4 :].reshape(h // 2, w // 2) - - texture.send_data((0, 0, 0), y) - texture.send_data((0, 0, 1), u) - texture.send_data((w // 2, 0, 1), v) - else: - # all other colorspaces can be directly sent - texture.send_data((0, 0, 0), chunk) + texture.send_data((0, 0, 0), chunk) self.buffer[buffer_index] = texture self._colorspace = colorspace - self._colorrange = colorrange self._cpu_buffer = cpu_buffer @property def colorspace( self, - ) -> Literal["srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"]: + ) -> ColorspacesRGB: """Colorspace, read only""" return self._colorspace - @property - def colorrange(self) -> Literal["full", "limited"]: - """Colorspace, read only property""" - return self._colorrange - @property def cpu_buffer(self) -> bool: """whether or not a cpu buffer exists for this TextureArray""" return self._cpu_buffer @property - def shape(self) -> tuple[int, ...]: + def shape(self) -> tuple[int, int] | tuple[int, int, int]: """ - the shape of the represented data - if the colorspace is yuv420p then it is the shape of the _packed_ data + the shape of the represented data, [n_rows, n_cols] or [n_rows, n_cols, 3 | 4] """ return self._shape @@ -244,31 +159,9 @@ def set_value(self, graphic, value: np.ndarray): # if cpu_buffer is False, we directly send data to the GPU if value.shape != self.shape: raise ValueError( - f"new data shape must be the same as the original data array" + f"new data shape must be the same as the original data array if `cpu_buffer=False`" f"original data shape was: {self.shape}, data passed is of shape: {value.shape}" ) - # send everything - if self.colorspace == "yuv420p": - if isinstance(value, (tuple, list)): - if not len(value) == 3: - raise ValueError - y, u, v = value - else: - # yuv data is packed. unpack, reshape and send with respective offsets - w = value.shape[1] - # u, v is packed in the bottom 1/3rd of rows - h = value.shape[0] * 2 // 3 - - y = value[:h] - u = value[h : h + h // 4].reshape(h // 2, w // 2) - v = value[h + h // 4 :].reshape(h // 2, w // 2) - - self._buffer[0, 0].send_data((0, 0, 0), y) - self._buffer[0, 0].send_data((0, 0, 1), u) - # width is y.shape[1] - self._buffer[0, 0].send_data((y.shape[1] // 2, 0, 1), v) - else: - # all other colorspaces can be directly sent for texture, buffer_index, slicer in self: chunk = value[slicer] texture.send_data((0, 0, 0), chunk) @@ -278,7 +171,7 @@ def set_value(self, graphic, value: np.ndarray): self[:] = value @property - def buffer(self) -> np.ndarray[pygfx.Texture]: + def buffer(self) -> NDArray[pygfx.Texture]: return self._buffer @property @@ -300,47 +193,20 @@ def col_indices(self) -> np.ndarray: def _check_data(self, data, colorspace, cpu_buffer): # make sure data ndim is valid for the given colorspace - if colorspace in ("srgb", "tex-srgb", "physical"): - if data.ndim not in (2, 3): - raise ValueError( - "if the colorspace is 'srgb', 'tex-srgb', or 'physical', " - "the image data must be 2D with or without an RGB(A) dimension, i.e. " - "it must be of shape [rows, cols], [rows, cols, 3] or [rows, cols, 4]" - ) - - if data.ndim == 3 and not cpu_buffer: - # wgpu only supports rgba, it does not support rgb - if data.shape[-1] != 4: - raise ValueError( - "if the colorspace is 'srgb', 'tex-srgb', or 'physical' and `cpu_buffer=False`" - "the image data MUST be RGBA, with shape [rows, cols, 4]. WGPU does not support " - "rgb textures. You must either supply full a RGBA array with `cpu_buffer=False` or " - "use `cpu_buffer=True` which supports RGB arrays." - ) - - elif colorspace == "yuv420p": - if isinstance(data, (tuple, list)): - if not len(data) == 3: - raise ValueError( - f"if `colorspace=yuv420p`, must provide a tuple/list of arrays representing y, u, v components" - f"(luma, and chroma channels), or a single arrays with packed yuv components. You passed: " - f"{data}" - ) - return data - # TODO: more validation for YUV expected dims - - elif data.ndim != 2: - raise ValueError( - "if the colorspace is 'yuv420p' the data array must have 2 dimensions, " - "with the `u` and `v` values packed along the bottom rows of the 2D data array" - ) + if data.ndim not in (2, 3): + raise ValueError( + "the image data must be 2D with or without an RGB(A) dimension, i.e. " + "it must be of shape [rows, cols], [rows, cols, 3] or [rows, cols, 4]" + ) - elif colorspace == "yuv444p": - if data.ndim != 3 or data.shape[-1] != 3: + if data.ndim == 3 and not cpu_buffer: + # wgpu only supports rgba, it does not support rgb + if data.shape[-1] != 4: raise ValueError( - "if the colorspace is 'yuv420p' the data array must have 3 dimensions, " - "the shape should be: [rows, cols, 3], i.e. a stack of 3 2D arrays that " - "represent y, u, v." + "if the colorspace is 'srgb', 'tex-srgb', or 'physical' and `cpu_buffer=False`" + "the image data MUST be RGBA, with shape [rows, cols, 4]. WGPU does not support " + "rgb textures. You must either supply full a RGBA array with `cpu_buffer=False` or " + "use `cpu_buffer=True` which supports RGB arrays." ) if data.itemsize == 8: @@ -411,6 +277,143 @@ def __len__(self): return self.buffer.size +class TextureYUV(GraphicFeature): + """ + Manages a YUV texture, no chunking + """ + + event_info_spec = [ + { + "dict key": "key", + "type": "slice, index, numpy-like fancy index", + "description": "key at which image data was sliced/fancy indexed", + }, + { + "dict key": "value", + "type": "np.ndarray | float", + "description": "new data values", + }, + ] + + def __init__( + self, + data: TupleYUV, + property_name: str = "data", + colorspace: ColorspacesYUV = ColorspacesYUV.yuv420p, + colorrrange: ColorRange = ColorRange.limited, + ): + super().__init__(property_name=property_name) + + self._colorspace = ColorspacesYUV(colorspace) + self._colorrange = ColorRange(colorrrange) + + self._check_data(data) + + self._data = data + + shared = pygfx.renderers.wgpu.get_shared() + limit = shared.device.limits["max-texture-dimension-2d"] + if data[0].shape[0] > limit or data[0].shape[1] > limit: + raise ValueError( + f"YUV colorspaces Images currently don't support dimensions that exceed the device's " + f"max-texture-dimension-2d. For now you must manually tile individual Images to use a YUV colorspace." + ) + + self._allocate_texture(data) + self._send_data(data) + + @property + def texture(self) -> pygfx.Texture: + return self._texture + + @property + def colorspace(self) -> ColorspacesYUV: + return self._colorspace + + @property + def colorrange(self) -> ColorRange: + return self._colorrange + + def _allocate_texture(self, data: TupleYUV): + self._h, self._w = data[0].shape + if self.colorspace == ColorspacesYUV.yuv420p: + depth = 2 + else: + depth = 3 + + self._texture = pygfx.Texture( + (self._w, self._h, depth), + dim=2, + colorspace=self.colorspace.value, + colorrange=self.colorrange.value, + format="r8unorm", + usage=wgpu.TextureUsage.COPY_DST, + ) + + def _send_data(self, data): + y, u, v = data + + self._texture.send_data((0, 0, 0), y) + + if self.colorspace == ColorspacesYUV.yuv420p: + self._texture.send_data((0, 0, 1), u) + self._texture.send_data((self._w // 2, 0, 1), v) + else: + self._texture.send_data((0, 0, 1), u) + self._texture.send_data((0, 0, 2), v) + + @property + def value(self) -> None: + """this is bufferless""" + return None + + def set_value(self, graphic, value: TupleYUV): + self._check_data(value) + + y, u, v = value + + if y.shape[0] != self._h or y.shape[1] != self._w: + self._allocate_texture(value) + graphic.geometry.grid = self._texture + + self._send_data(value) + + def _check_data(self, data: TupleYUV): + err = f"must provide a tuple/list of np.ndarray of type np.uint8 representing YUV components." + + if not isinstance(data, (tuple, list)): + raise TypeError(err + f"\nYou provided: {data}") + + if not len(data) == 3: + raise TypeError(err + f"\nYou provided data of len: {len(data)}") + + if not all([isinstance(a, np.ndarray) for a in data]): + raise TypeError(err + f"\nYou provided types: {[type(d) for d in data]}") + + types = [a.dtype for a in data] + if not all([t == np.uint8 for t in types]): + raise TypeError(err + f"\nYou provided data of types: {types}") + + if self.colorspace == ColorspacesYUV.yuv420p: + err += ( + f"For {self.colorspace} UV channels must be 4x smaller than Y. " + f"You provided shapes: {(d.shape for d in data)}" + ) + shapes = tuple(np.asarray(d.shape) for d in data) + expected_uv_shape = shapes[0] // 4 + if shapes[1] != expected_uv_shape or shapes[2] != expected_uv_shape: + raise ValueError(err) + + else: + err += ( + f"For {self.colorspace} UV channels must be the same size as Y" + f"You provided shapes: {(d.shape for d in data)}" + ) + + if data[0].shape != data[1].shape or data[0].shape != data[2].shape: + raise ValueError(err) + + class ImageVmin(GraphicFeature): """lower contrast limit""" diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index fe2e75082..db77330ac 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -5,7 +5,7 @@ import pygfx from pygfx import Texture -from ..utils import quick_min_max +from ..utils import quick_min_max, ColorspacesRGB, ColorspacesYUV, ColorRange from ._base import Graphic from .selectors import ( LinearSelector, @@ -15,6 +15,8 @@ ) from .features import ( TextureArray, + TextureYUV, + TupleYUV, ImageCmap, ImageVmin, ImageVmax, @@ -104,21 +106,17 @@ def __init__( cmap: str = "plasma", interpolation: str = "nearest", cmap_interpolation: str = "linear", - colorspace: Literal[ - "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p" - ] = "srgb", - colorrange: Literal["full", "limited"] = "full", + colorspace: ColorspacesRGB = "srgb", cpu_buffer: bool = True, **kwargs, ): """ - Create an Image Graphic + Create an ImageGraphic Parameters ---------- data: array-like array-like, usually numpy.ndarray, must support ``memoryview()`` - # TODO: update this, and also allow tuple/list of arrays for yuv420p | shape must be ``[n_rows, n_cols]``, ``[n_rows, n_cols, 3]`` for RGB or ``[n_rows, n_cols, 4]`` for RGBA vmin: float, optional @@ -137,7 +135,7 @@ def __init__( cmap_interpolation: str, optional, default "linear" colormap interpolation method, one of "nearest" or "linear" - colorspace: one of "srgb", "tex-srgb", "physical", "yuv420p", "yuv444p", default "srgb" + colorspace: one of "srgb", "tex-srgb", "physical", default "srgb" colorspace in which to interpret the provided data. * "srgb": the data represents intensity, rgb, or rgba pixels in the sRGB space. @@ -154,47 +152,6 @@ def __init__( * "physical": the colors are (already) in the physical / linear space, where lighting calculations can be applied. Shader code that interprets the data as color will use it as-is. - * "yuv420p": A common video format. The data is represented as 3 planes (y, u, and v). - The y represents intensity, and is at full resolution. The u and v planes are a - quarter of the size. data must be a 2D array which packs y, u and v: - - ====== - | y | - | . | - | . | - | . | - ------ - | u | - ------ - | v | - ====== - - This is the same as the packed array structure that pyav provides when reading video in as yuv420p. - - If the data represents an image with width and height (w, h), then the packed data array must be of - shape: [w, h * 3 // 2]. - - # TODO: You can also provide a tuple of arrays to data: (y, u, v) - - For more info see: https://docs.pygfx.org/stable/_gallery/feature_demo/video_yuv.html - and https://github.com/pygfx/pygfx/pull/873 - - - * "yuv444p": A lesser common video format. The data is represented as 3 planes - (y, u, and v) similar to yuv420p however the u and v planes are stored - at full resolution. - - colorrange: Literal["full", "limited"] = "limited", - Relevant for yuv colorspaces. Most videos use "limited". - - * "limited": The luma plane (Y) is limited to the range of 16-235 for 8 bits. - The chroma planes (U and V) are limited to the range of 16-240 for 8 bits - * "full": The luma plane and chroma plane use the full range of the storage format. - - See the following links from the FFMPEG documentation for more details: - https://trac.ffmpeg.org/wiki/colorspace - https://ffmpeg.org/doxygen/7.0/pixfmt_8h_source.html#l00609 - cpu_buffer: bool, default True If ``True``, maintains a buffer of system RAM that is sychronized with a corresponding storage buffer on the GPU. @@ -207,6 +164,8 @@ def __init__( * tooltip values for grayscale data are estimated using an inverse transforms on the colormap LUT. The tooltip values may or may not be accurate for a given colormap and vmin, vmax. If you require precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. + * vmin, vmax must be explicitly provided if sharing an existing buffer from another ImageGraphic + * ``reset_vmin_vmax()`` is not supported kwargs: additional keyword arguments passed to :class:`.Graphic` @@ -217,32 +176,24 @@ def __init__( group = pygfx.Group() - self._colorspace = colorspace - self._colorrange = colorrange - if isinstance(data, TextureArray): # share buffer self._data = data + data = self._data.value else: # create new texture array to manage buffer # texture array that manages the multiple textures on the GPU that represent this image - self._data = TextureArray( - data, - cpu_buffer=cpu_buffer, - colorspace=colorspace, - colorrange=colorrange, - ) - - if isinstance(data, (tuple, list)): - # unpacked yuv - data = data[0] + self._data = TextureArray(data, cpu_buffer) - if (vmin is None) or (vmax is None): - _vmin, _vmax = quick_min_max(data) + if (vmin is None) or (vmax is None) and data is not None: + _vmin, _vmax = quick_min_max(self.data.value) if vmin is None: vmin = _vmin if vmax is None: vmax = _vmax + else: + # this is a shared buffer and we don't have access to the actual data from here + raise ValueError("must provide vmin, vmax if sharing a buffer that does not exist locally") # other graphic features self._vmin = ImageVmin(vmin) @@ -251,11 +202,12 @@ def __init__( self._interpolation = ImageInterpolation(interpolation) self._cmap_interpolation = ImageCmapInterpolation(cmap_interpolation) - # cmap only used for grayscale images - self._cmap = None - _map = None + # set map to None for RGB images + if len(self.data.shape) == 3: + self._cmap = None + _map = None - if data.ndim == 2 and colorspace != "yuv420p": + else: # use TextureMap for grayscale images self._cmap = ImageCmap(cmap) @@ -305,6 +257,11 @@ def _create_tiles(self) -> list[_ImageTile]: return tiles + @property + def cpu_buffer(self) -> bool: + """whether or not a cpu buffer is used for the image data. If ``False``, then the data only exist on the GPU""" + return self.data.cpu_buffer + @property def data(self) -> TextureArray: """ @@ -312,34 +269,28 @@ def data(self) -> TextureArray: Note that if the shape of the new data array does not equal the shape of current data array, a new set of GPU Textures are automatically created. - This can have performance drawbacks when you have a very large image. + This can have performance drawbacks when you have a ver large images. This is usually fine as long as you don't need to do it hundreds of times per second. """ return self._data @data.setter - def data(self, new_data): - if isinstance(new_data, np.ndarray): + def data(self, data): + if isinstance(data, np.ndarray): # check if a new buffer is required - if self._data.shape != new_data.shape: + if self._data.value.shape != data.shape: # create new TextureArray - self._data = TextureArray( - new_data, - cpu_buffer=self.cpu_buffer, - colorspace=self.colorspace, - colorrange=self.colorrange, - ) + self._data = TextureArray(data) - # see if the new texture data needs a cmap - if len(self._data.shape) == 3 and self._data.colorspace != "yuv420p": - # set cmap to None since data is not grayscale + # cmap based on if rgb or grayscale + if self._data.value.ndim > 2: self._cmap = None + + # must be None if RGB(A) self._material.map = None else: - if ( - self.cmap is None - ): # have switched from non-grayscale -> grayscale image + if self.cmap is None: # have switched from RGBA -> grayscale image # create default cmap self._cmap = ImageCmap("plasma") self._material.map = pygfx.TextureMap( @@ -363,24 +314,13 @@ def data(self, new_data): return - self._data.set_value(self, new_data) - - @property - def cpu_buffer(self) -> bool: - return self.data.cpu_buffer + self._data[:] = data @property - def colorspace( - self, - ) -> Literal["srgb", "tex-srgb", "physical", "yuv420p", "yuv444p"]: - """colorspace, read-only property""" + def colorspace(self) -> ColorspacesRGB: + """The image's colorspace""" return self.data.colorspace - @property - def colorrange(self) -> Literal["full", "limited"]: - """colorrange, read-only property""" - return self.data.colorrange - @property def cmap(self) -> str | None: """ @@ -393,9 +333,8 @@ def cmap(self) -> str | None: @cmap.setter def cmap(self, name: str): - if len(self.data.shape) > 2: - raise AttributeError("cmap is only supported for grayscale images") - + if self.data.value.ndim > 2: + raise AttributeError("RGB(A) images do not have a colormap property") self._cmap.set_value(self, name) @property @@ -438,15 +377,15 @@ def reset_vmin_vmax(self): """ Reset the vmin, vmax by estimating it from the data by subsampling. """ - if not self.cpu_buffer: - return + if self.data.value is None: + raise NotImplemented("Cannot reset vmin, vmax if `cpu_buffer=False`") vmin, vmax = quick_min_max(self._data.value) self.vmin = vmin self.vmax = vmax def add_linear_selector( - self, selection: int = None, axis: str = "x", **kwargs + self, selection: int = None, axis: str = "x", **kwargs ) -> LinearSelector: """ Adds a :class:`.LinearSelector`. @@ -496,12 +435,12 @@ def add_linear_selector( return selector def add_linear_region_selector( - self, - selection: tuple[float, float] = None, - axis: str = "x", - padding: float = 0.0, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, + self, + selection: tuple[float, float] = None, + axis: str = "x", + padding: float = 0.0, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, ) -> LinearRegionSelector: """ Add a :class:`.LinearRegionSelector`. @@ -571,10 +510,10 @@ def add_linear_region_selector( return selector def add_rectangle_selector( - self, - selection: tuple[float, float, float, float] = None, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, + self, + selection: tuple[float, float, float, float] = None, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, ) -> RectangleSelector: """ Add a :class:`.RectangleSelector`. @@ -613,10 +552,10 @@ def add_rectangle_selector( return selector def add_polygon_selector( - self, - selection: List[tuple[float, float]] = None, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, + self, + selection: List[tuple[float, float]] = None, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, ) -> PolygonSelector: """ Add a :class:`.PolygonSelector`. @@ -677,3 +616,140 @@ def format_pick_info(self, pick_info: dict) -> str: ) return info + +class ImageYUVGraphic(ImageGraphic): + _features = { + "data": TextureYUV, + "vmin": ImageVmin, + "vmax": ImageVmax, + "interpolation": ImageInterpolation, + } + + def __init__( + self, + data: TupleYUV | TextureYUV, + vmin: float = 0, + vmax: float = 255, + interpolation: str = "nearest", + colorspace: ColorspacesYUV = "yuv420p", + colorrange: ColorRange = "limited", + **kwargs, + ): + """ + Create an ImageYUVGraphic. Similar to ImageGraphic but handles data that is in yuv42p or yuv444p colorspace. + + Note that the buffers for YUV Images only exist on the GPU. When setting the image data, the new values are + directly sent to the GPU. + + ``reset_vmin_vmax()`` just sets (vmin, vmax) to (0, 255) + + Parameters + ---------- + data: TupleYUV + tuple of arrays that represent YUV channels. If the colorspace is yuv420p, the U and V array dims + must be 4 times smaller than the Y array dims. + + vmin: float, optional, default 0 + minimum value for color scaling + + vmax: float, optional, default 255 + maximum value for color scaling + + interpolation: str, optional, default "nearest" + interpolation filter, one of "nearest" or "linear" + + colorspace: "yuv42p" | "yuv444p" + colorspace in which to interpret the provided data. + + * "yuv420p": A common video format. The data is represented as 3 planes (y, u, and v). + The y represents intensity, and is at full resolution. The u and v planes are a + quarter of the size. + + * "yuv444p": A lesser common video format. The data is represented as 3 planes + (y, u, and v) similar to yuv420p however the u and v planes are stored + at full resolution. + + colorrange: Literal["full", "limited"] = "limited", + Relevant for yuv colorspaces. Most videos use "limited". + + * "limited": The luma plane (Y) is limited to the range of 16-235 for 8 bits. + The chroma planes (U and V) are limited to the range of 16-240 for 8 bits + * "full": The luma plane and chroma plane use the full range of the storage format. + + See the following links from the FFMPEG documentation for more details: + https://trac.ffmpeg.org/wiki/colorspace + https://ffmpeg.org/doxygen/7.0/pixfmt_8h_source.html#l00609 + + cpu_buffer: bool, default True + If ``True``, maintains a buffer of system RAM that is sychronized with a corresponding storage buffer + on the GPU. + If ``False``, setting the graphic data will send the new data directly to the GPU, we also + call this "bufferless". This is much faster but lacks the following features: + * you must update the entire data array, i.e. you can perform ``image.data = new_data``, and you + cannot perform partial updates such as ``image.data[indices] = ``. + * RGB arrays of shape [rows, cols, 3] are not supported since wgpu does not have RGB textures, + use RGBA or use `cpu_buffer=True` if you really need RGB instead of RGBA. + * tooltip values for grayscale data are estimated using an inverse transforms on the colormap LUT. + The tooltip values may or may not be accurate for a given colormap and vmin, vmax. If you require + precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. + + kwargs: + additional keyword arguments passed to :class:`.Graphic` + + """ + super().__init__(**kwargs) + + if isinstance(data, TextureYUV): + # share buffer + self._data = data + else: + self._data = TextureYUV(data, colorspace) + + self._vmin = ImageVmin(vmin) + self._vmax = ImageVmax(vmax) + + self._interpolation = ImageInterpolation(interpolation) + + self._material = pygfx.ImageBasicMaterial( + clim=(vmin, vmax), + interpolation=self.interpolation.value, + pick_write=True + ) + + wo = pygfx.Image( + geometry=pygfx.Geometry(grid=self.data._texture), + material=self._material, + ) + + self._set_world_object(wo) + + @property + def data(self) -> TextureYUV: + """ + YUV Texture data, note that no local buffer exists for YUV images, you can only set values but not get them + """ + return self._data + + @data.setter + def data(self, data): + self.data.set_value(self, data) + + @property + def colorspace(self) -> ColorspacesYUV: + """image's colorspace""" + return self.data.colorspace + + @property + def colorrange(self) -> ColorRange: + return self.data.colorrange + + @property + def cmap(self): + raise NotImplemented("YUV images don't have a cmap") + + @property + def cmap_interpolation(self): + raise NotRequired("YUV images don't have a cmap") + + def reset_vmin_vmax(self): + self.vmin, self.vmax = 0, 255 diff --git a/fastplotlib/utils/enums.py b/fastplotlib/utils/enums.py index 3901b082c..44601350d 100644 --- a/fastplotlib/utils/enums.py +++ b/fastplotlib/utils/enums.py @@ -1,4 +1,4 @@ -from enum import IntEnum +from enum import IntEnum, StrEnum class RenderQueue(IntEnum): @@ -13,3 +13,19 @@ class RenderQueue(IntEnum): # the graphics. Axes (rulers) have depth_compare '<=' and selectors don't compare depth. axes = 3400 # still in 'object' group selector = 3600 # considered in 'overlay' group + + +class ColorspacesRGB(StrEnum): + srgb = "srgb" + tex_srgb = "tex-srgb" + physical = "physical" + + +class ColorspacesYUV(StrEnum): + yuv420p = "yuv420p" + yuv444p = "yuv444p" + + +class ColorRange(StrEnum): + full = "full" + limited = "limited" From 3eb286930fcf14bddcaaf90fb6518c34eca06ddc Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 18:15:59 -0400 Subject: [PATCH 09/18] docstrings --- fastplotlib/graphics/image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index db77330ac..cac6bdd2c 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -741,6 +741,7 @@ def colorspace(self) -> ColorspacesYUV: @property def colorrange(self) -> ColorRange: + """the color range, see docstring for details""" return self.data.colorrange @property @@ -752,4 +753,5 @@ def cmap_interpolation(self): raise NotRequired("YUV images don't have a cmap") def reset_vmin_vmax(self): + """reset vmin, vmax to (0, 255)""" self.vmin, self.vmax = 0, 255 From 537c3849e5e0a74ead5a5a9c2b3322c6d9504684 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 18:17:12 -0400 Subject: [PATCH 10/18] docstrings --- fastplotlib/graphics/features/_image.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 7ae0c2a75..297779b16 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -279,7 +279,7 @@ def __len__(self): class TextureYUV(GraphicFeature): """ - Manages a YUV texture, no chunking + Manages a YUV texture, no chunking, no local buffer """ event_info_spec = [ @@ -335,6 +335,8 @@ def colorrange(self) -> ColorRange: return self._colorrange def _allocate_texture(self, data: TupleYUV): + """Create a new pygfx.Texture""" + self._h, self._w = data[0].shape if self.colorspace == ColorspacesYUV.yuv420p: depth = 2 @@ -351,6 +353,7 @@ def _allocate_texture(self, data: TupleYUV): ) def _send_data(self, data): + """send the data to the GPU""" y, u, v = data self._texture.send_data((0, 0, 0), y) From 3f4934599f719390128eda7e5eff7d246101fbd4 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 18:23:58 -0400 Subject: [PATCH 11/18] import order --- fastplotlib/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastplotlib/utils/__init__.py b/fastplotlib/utils/__init__.py index f2eed65b6..cb6a240d1 100644 --- a/fastplotlib/utils/__init__.py +++ b/fastplotlib/utils/__init__.py @@ -2,10 +2,10 @@ # this MUST be imported as early as possible in fpl.__init__ before any other wgpu stuff from .gui import loop +from .enums import * from .functions import * from .gpu import enumerate_adapters, select_adapter, print_wgpu_report from ._plot_helpers import * -from .enums import * from .protocols import ARRAY_LIKE_ATTRS, ArrayProtocol, FutureProtocol, CudaArrayProtocol From 08867f4801950613c5c6e7fbb9770e1769ba7156 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 19:33:44 -0400 Subject: [PATCH 12/18] yuv graphic working nicely --- examples/image/image_yuv.py | 45 ++ fastplotlib/graphics/__init__.py | 3 +- fastplotlib/graphics/features/_image.py | 17 +- fastplotlib/graphics/image.py | 550 +++++++++--------- .../graphics/selectors/_linear_region.py | 6 + fastplotlib/graphics/selectors/_polygon.py | 6 + fastplotlib/graphics/selectors/_rectangle.py | 6 + fastplotlib/layouts/_figure.py | 4 +- fastplotlib/layouts/_graphic_methods_mixin.py | 132 ++++- 9 files changed, 487 insertions(+), 282 deletions(-) create mode 100644 examples/image/image_yuv.py diff --git a/examples/image/image_yuv.py b/examples/image/image_yuv.py new file mode 100644 index 000000000..a4790a32d --- /dev/null +++ b/examples/image/image_yuv.py @@ -0,0 +1,45 @@ +""" +YUV Image +========= + +Example that shows how to use YUV images. Most videos are stored in this colorspace. +Y stores luma at full resolution, UV stores chroma values. +In yuv420p UV channels are stored at half the resolution of Y. In yuv444p, UV channels are stored +at full resolution. + +YUV is also called YCbCr for digital images. + +For more info: https://en.wikipedia.org/wiki/Y%E2%80%B2UV +""" + +# test_example = true +# sphinx_gallery_pygfx_docs = 'screenshot' + +import fastplotlib as fpl +import numpy as np +from skimage.color import rgb2ycbcr +import imageio.v3 as iio + +# convert an rgb image to ycbcr for example purposes +img = iio.imread("imageio:astronaut.png") +img_yuv = rgb2ycbcr(img).astype(np.uint8) + +y = img_yuv[..., 0] +u = img_yuv[::2, ::2, 1] +v = img_yuv[::2, ::2, 2] + +figure = fpl.Figure(size=(700, 560)) + +# plot the image data +image = figure[0, 0].add_image_yuv( + data=(y, u, v), colorspace="yuv420p", name="yuv image" +) + +figure.show() + + +# NOTE: fpl.loop.run() should not be used for interactive sessions +# See the "JupyterLab and IPython" section in the user guide +if __name__ == "__main__": + print(__doc__) + fpl.loop.run() diff --git a/fastplotlib/graphics/__init__.py b/fastplotlib/graphics/__init__.py index cca2afc21..baf8151be 100644 --- a/fastplotlib/graphics/__init__.py +++ b/fastplotlib/graphics/__init__.py @@ -1,7 +1,7 @@ from ._base import Graphic from .line import LineGraphic from .scatter import ScatterGraphic -from .image import ImageGraphic +from .image import ImageGraphic, ImageYUVGraphic from .image_volume import ImageVolumeGraphic from ._vectors import VectorsGraphic from .mesh import MeshGraphic, SurfaceGraphic, PolygonGraphic @@ -14,6 +14,7 @@ "LineGraphic", "ScatterGraphic", "ImageGraphic", + "ImageYUVGraphic", "ImageVolumeGraphic", "VectorsGraphic", "MeshGraphic", diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index 297779b16..fba629ebe 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -77,7 +77,6 @@ def __init__( self._shape = data.shape - # data start indices for each Texture self._row_indices = np.arange( 0, @@ -322,6 +321,10 @@ def __init__( self._allocate_texture(data) self._send_data(data) + @property + def cpu_buffer(self) -> Literal[False]: + return False + @property def texture(self) -> pygfx.Texture: return self._texture @@ -344,7 +347,7 @@ def _allocate_texture(self, data: TupleYUV): depth = 3 self._texture = pygfx.Texture( - (self._w, self._h, depth), + size=(self._w, self._h, depth), dim=2, colorspace=self.colorspace.value, colorrange=self.colorrange.value, @@ -400,17 +403,19 @@ def _check_data(self, data: TupleYUV): if self.colorspace == ColorspacesYUV.yuv420p: err += ( f"For {self.colorspace} UV channels must be 4x smaller than Y. " - f"You provided shapes: {(d.shape for d in data)}" + f"You provided shapes: {tuple(d.shape for d in data)}" ) shapes = tuple(np.asarray(d.shape) for d in data) - expected_uv_shape = shapes[0] // 4 - if shapes[1] != expected_uv_shape or shapes[2] != expected_uv_shape: + expected_uv_shape = shapes[0] // 2 + if (shapes[1] != expected_uv_shape).all() or ( + shapes[2] != expected_uv_shape + ).all(): raise ValueError(err) else: err += ( f"For {self.colorspace} UV channels must be the same size as Y" - f"You provided shapes: {(d.shape for d in data)}" + f"You provided shapes: {tuple(d.shape for d in data)}" ) if data[0].shape != data[1].shape or data[0].shape != data[2].shape: diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index cac6bdd2c..aff0927f5 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -88,7 +88,274 @@ def chunk_index(self) -> tuple[int, int]: return self._chunk_index -class ImageGraphic(Graphic): +class ImageBase(Graphic): + @property + def cpu_buffer(self) -> bool: + """whether or not a cpu buffer is used for the image data. If ``False``, then the data only exist on the GPU""" + return self.data.cpu_buffer + + @property + def vmin(self) -> float: + """lower contrast limit""" + return self._vmin.value + + @vmin.setter + def vmin(self, value: float): + self._vmin.set_value(self, value) + + @property + def vmax(self) -> float: + """upper contrast limit""" + return self._vmax.value + + @vmax.setter + def vmax(self, value: float): + self._vmax.set_value(self, value) + + @property + def interpolation(self) -> str: + """Data interpolation method""" + return self._interpolation.value + + @interpolation.setter + def interpolation(self, value: str): + self._interpolation.set_value(self, value) + + def add_linear_selector( + self, selection: int = None, axis: str = "x", **kwargs + ) -> LinearSelector: + """ + Adds a :class:`.LinearSelector`. + + Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them + from a plot area just like any other ``Graphic``. + + Parameters + ---------- + selection: int, optional + initial position of the selector + + kwargs: + passed to :class:`.LinearSelector` + + Returns + ------- + LinearSelector + + """ + + if axis == "x": + limits = (0, self._data.value.shape[1]) + elif axis == "y": + limits = (0, self._data.value.shape[0]) + else: + raise ValueError("`axis` must be one of 'x' | 'y'") + + if selection is None: + selection = limits[0] + + if selection < limits[0] or selection > limits[1]: + raise ValueError( + f"the passed selection: {selection} is beyond the limits: {limits}" + ) + + selector = LinearSelector( + selection=selection, + limits=limits, + axis=axis, + parent=self, + **kwargs, + ) + + self._plot_area.add_graphic(selector, center=False) + + return selector + + def add_linear_region_selector( + self, + selection: tuple[float, float] = None, + axis: str = "x", + padding: float = 0.0, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, + ) -> LinearRegionSelector: + """ + Add a :class:`.LinearRegionSelector`. + + Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them + from a plot area just like any other ``Graphic``. + + Parameters + ---------- + selection: (float, float) + initial (min, max) of the selection + + axis: "x" | "y" + axis the selector can move along + + padding: float, default 100.0 + Extends the linear selector along the perpendicular axis to make it easier to interact with. + + kwargs + passed to ``LinearRegionSelector`` + + Returns + ------- + LinearRegionSelector + + """ + + if axis == "x": + size = self._data.value.shape[0] + center = size / 2 + limits = (0, self._data.value.shape[1]) + elif axis == "y": + size = self._data.value.shape[1] + center = size / 2 + limits = (0, self._data.value.shape[0]) + else: + raise ValueError("`axis` must be one of 'x' | 'y'") + + # default padding is 25% the height or width of the image + if padding is None: + size *= 1.25 + else: + size += padding + + if selection is None: + selection = limits[0], int(limits[1] * 0.25) + + if padding is None: + size *= 1.25 + + else: + size += padding + + selector = LinearRegionSelector( + selection=selection, + limits=limits, + size=size, + center=center, + axis=axis, + fill_color=fill_color, + parent=self, + **kwargs, + ) + + self._plot_area.add_graphic(selector, center=False) + + return selector + + def add_rectangle_selector( + self, + selection: tuple[float, float, float, float] = None, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, + ) -> RectangleSelector: + """ + Add a :class:`.RectangleSelector`. + + Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them + from a plot area just like any other ``Graphic``. + + Parameters + ---------- + selection: (float, float, float, float), optional + initial (xmin, xmax, ymin, ymax) of the selection + + """ + # default selection is 25% of the diagonal + if selection is None: + diagonal = math.sqrt( + self._data.value.shape[0] ** 2 + self._data.value.shape[1] ** 2 + ) + + selection = (0, int(diagonal / 4), 0, int(diagonal / 4)) + + # min/max limits are image shape + # rows are ys, columns are xs + limits = (0, self._data.value.shape[1], 0, self._data.value.shape[0]) + + selector = RectangleSelector( + selection=selection, + limits=limits, + fill_color=fill_color, + parent=self, + **kwargs, + ) + + self._plot_area.add_graphic(selector, center=False) + + return selector + + def add_polygon_selector( + self, + selection: List[tuple[float, float]] = None, + fill_color=(0, 0, 0.35, 0.2), + **kwargs, + ) -> PolygonSelector: + """ + Add a :class:`.PolygonSelector`. + + Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them + from a plot area just like any other ``Graphic``. + + Parameters + ---------- + selection: list[tuple[float, float]], optional + Initial points for the polygon. If not given or None, you'll start drawing the selection (clicking adds points to the polygon). + + """ + + # min/max limits are image shape + # rows are ys, columns are xs + limits = (0, self._data.value.shape[1], 0, self._data.value.shape[0]) + + selector = PolygonSelector( + selection, + limits, + fill_color=fill_color, + parent=self, + **kwargs, + ) + + self._plot_area.add_graphic(selector, center=False) + + return selector + + def format_pick_info(self, pick_info: dict) -> str: + if not self.cpu_buffer: + if self.colorspace not in ColorspacesYUV and len(self.data.shape) == 2: + # inverse map from rgb pixel value to grayscale value using the colormap + # we can only perform a guess + lut = self._material.map.texture.data + rgb = pick_info["rgba"][:3] + closest = np.argmin(np.linalg.norm(lut[:, :3] - rgb, axis=1)) + scalar = closest / (lut.shape[0] - 1) + val = self.vmin + scalar * (self.vmax - self.vmin) + return f"{val:.4g}\n!!estimate!!, cpu_buffer=False" + else: + # direct rgba vals + rgba_val = pick_info["rgba"] + info = "\n".join( + f"{channel}: {val: .4g}" for channel, val in zip("rgba", rgba_val) + ) + return info + + col, row = pick_info["index"] + if self.data.value.ndim == 2: + val = self.data[row, col] + info = f"{val:.4g}" + else: + info = "\n".join( + f"{channel}: {val:.4g}" + for channel, val in zip("rgba", self.data[row, col]) + ) + + return info + + +class ImageGraphic(ImageBase): _features = { "data": TextureArray, "cmap": ImageCmap, @@ -166,6 +433,7 @@ def __init__( precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. * vmin, vmax must be explicitly provided if sharing an existing buffer from another ImageGraphic * ``reset_vmin_vmax()`` is not supported + * selector tools will not be able to return the data under the selection kwargs: additional keyword arguments passed to :class:`.Graphic` @@ -183,7 +451,9 @@ def __init__( else: # create new texture array to manage buffer # texture array that manages the multiple textures on the GPU that represent this image - self._data = TextureArray(data, cpu_buffer) + self._data = TextureArray( + data, colorspace=colorspace, cpu_buffer=cpu_buffer + ) if (vmin is None) or (vmax is None) and data is not None: _vmin, _vmax = quick_min_max(self.data.value) @@ -193,7 +463,9 @@ def __init__( vmax = _vmax else: # this is a shared buffer and we don't have access to the actual data from here - raise ValueError("must provide vmin, vmax if sharing a buffer that does not exist locally") + raise ValueError( + "must provide vmin, vmax if sharing a buffer that does not exist locally" + ) # other graphic features self._vmin = ImageVmin(vmin) @@ -257,11 +529,6 @@ def _create_tiles(self) -> list[_ImageTile]: return tiles - @property - def cpu_buffer(self) -> bool: - """whether or not a cpu buffer is used for the image data. If ``False``, then the data only exist on the GPU""" - return self.data.cpu_buffer - @property def data(self) -> TextureArray: """ @@ -337,33 +604,6 @@ def cmap(self, name: str): raise AttributeError("RGB(A) images do not have a colormap property") self._cmap.set_value(self, name) - @property - def vmin(self) -> float: - """lower contrast limit""" - return self._vmin.value - - @vmin.setter - def vmin(self, value: float): - self._vmin.set_value(self, value) - - @property - def vmax(self) -> float: - """upper contrast limit""" - return self._vmax.value - - @vmax.setter - def vmax(self, value: float): - self._vmax.set_value(self, value) - - @property - def interpolation(self) -> str: - """Data interpolation method""" - return self._interpolation.value - - @interpolation.setter - def interpolation(self, value: str): - self._interpolation.set_value(self, value) - @property def cmap_interpolation(self) -> str: """cmap interpolation method, 'linear' or 'nearest'. Used only for grayscale images""" @@ -384,240 +624,8 @@ def reset_vmin_vmax(self): self.vmin = vmin self.vmax = vmax - def add_linear_selector( - self, selection: int = None, axis: str = "x", **kwargs - ) -> LinearSelector: - """ - Adds a :class:`.LinearSelector`. - - Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them - from a plot area just like any other ``Graphic``. - - Parameters - ---------- - selection: int, optional - initial position of the selector - - kwargs: - passed to :class:`.LinearSelector` - - Returns - ------- - LinearSelector - - """ - - if axis == "x": - limits = (0, self._data.value.shape[1]) - elif axis == "y": - limits = (0, self._data.value.shape[0]) - else: - raise ValueError("`axis` must be one of 'x' | 'y'") - - if selection is None: - selection = limits[0] - - if selection < limits[0] or selection > limits[1]: - raise ValueError( - f"the passed selection: {selection} is beyond the limits: {limits}" - ) - - selector = LinearSelector( - selection=selection, - limits=limits, - axis=axis, - parent=self, - **kwargs, - ) - - self._plot_area.add_graphic(selector, center=False) - - return selector - - def add_linear_region_selector( - self, - selection: tuple[float, float] = None, - axis: str = "x", - padding: float = 0.0, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, - ) -> LinearRegionSelector: - """ - Add a :class:`.LinearRegionSelector`. - - Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them - from a plot area just like any other ``Graphic``. - - Parameters - ---------- - selection: (float, float) - initial (min, max) of the selection - - axis: "x" | "y" - axis the selector can move along - - padding: float, default 100.0 - Extends the linear selector along the perpendicular axis to make it easier to interact with. - - kwargs - passed to ``LinearRegionSelector`` - - Returns - ------- - LinearRegionSelector - - """ - - if axis == "x": - size = self._data.value.shape[0] - center = size / 2 - limits = (0, self._data.value.shape[1]) - elif axis == "y": - size = self._data.value.shape[1] - center = size / 2 - limits = (0, self._data.value.shape[0]) - else: - raise ValueError("`axis` must be one of 'x' | 'y'") - - # default padding is 25% the height or width of the image - if padding is None: - size *= 1.25 - else: - size += padding - - if selection is None: - selection = limits[0], int(limits[1] * 0.25) - - if padding is None: - size *= 1.25 - - else: - size += padding - - selector = LinearRegionSelector( - selection=selection, - limits=limits, - size=size, - center=center, - axis=axis, - fill_color=fill_color, - parent=self, - **kwargs, - ) - - self._plot_area.add_graphic(selector, center=False) - - return selector - - def add_rectangle_selector( - self, - selection: tuple[float, float, float, float] = None, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, - ) -> RectangleSelector: - """ - Add a :class:`.RectangleSelector`. - - Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them - from a plot area just like any other ``Graphic``. - - Parameters - ---------- - selection: (float, float, float, float), optional - initial (xmin, xmax, ymin, ymax) of the selection - - """ - # default selection is 25% of the diagonal - if selection is None: - diagonal = math.sqrt( - self._data.value.shape[0] ** 2 + self._data.value.shape[1] ** 2 - ) - - selection = (0, int(diagonal / 4), 0, int(diagonal / 4)) - - # min/max limits are image shape - # rows are ys, columns are xs - limits = (0, self._data.value.shape[1], 0, self._data.value.shape[0]) - - selector = RectangleSelector( - selection=selection, - limits=limits, - fill_color=fill_color, - parent=self, - **kwargs, - ) - - self._plot_area.add_graphic(selector, center=False) - return selector - - def add_polygon_selector( - self, - selection: List[tuple[float, float]] = None, - fill_color=(0, 0, 0.35, 0.2), - **kwargs, - ) -> PolygonSelector: - """ - Add a :class:`.PolygonSelector`. - - Selectors are just ``Graphic`` objects, so you can manage, remove, or delete them - from a plot area just like any other ``Graphic``. - - Parameters - ---------- - selection: list[tuple[float, float]], optional - Initial points for the polygon. If not given or None, you'll start drawing the selection (clicking adds points to the polygon). - - """ - - # min/max limits are image shape - # rows are ys, columns are xs - limits = (0, self._data.value.shape[1], 0, self._data.value.shape[0]) - - selector = PolygonSelector( - selection, - limits, - fill_color=fill_color, - parent=self, - **kwargs, - ) - - self._plot_area.add_graphic(selector, center=False) - - return selector - - def format_pick_info(self, pick_info: dict) -> str: - if not self.cpu_buffer: - if self.data.colorspace != "yuv420p" and len(self.data.shape) == 2: - # inverse map from rgb pixel value to grayscale value using the colormap - # we can only perform a guess - lut = self._material.map.texture.data - rgb = pick_info["rgba"][:3] - closest = np.argmin(np.linalg.norm(lut[:, :3] - rgb, axis=1)) - scalar = closest / (lut.shape[0] - 1) - val = self.vmin + scalar * (self.vmax - self.vmin) - return f"{val:.4g}\n!!estimate!!, cpu_buffer=False" - else: - # rgba vals - rgba_val = pick_info["rgba"] - info = "\n".join( - f"{channel}: {val: .4g}" for channel, val in zip("rgba", rgba_val) - ) - return info - - col, row = pick_info["index"] - if self.data.value.ndim == 2: - val = self.data[row, col] - info = f"{val:.4g}" - else: - info = "\n".join( - f"{channel}: {val:.4g}" - for channel, val in zip("rgba", self.data[row, col]) - ) - - return info - -class ImageYUVGraphic(ImageGraphic): +class ImageYUVGraphic(ImageBase): _features = { "data": TextureYUV, "vmin": ImageVmin, @@ -703,7 +711,7 @@ def __init__( # share buffer self._data = data else: - self._data = TextureYUV(data, colorspace) + self._data = TextureYUV(data, colorspace=colorspace) self._vmin = ImageVmin(vmin) self._vmax = ImageVmax(vmax) @@ -711,9 +719,7 @@ def __init__( self._interpolation = ImageInterpolation(interpolation) self._material = pygfx.ImageBasicMaterial( - clim=(vmin, vmax), - interpolation=self.interpolation.value, - pick_write=True + clim=(vmin, vmax), interpolation=self.interpolation, pick_write=True ) wo = pygfx.Image( diff --git a/fastplotlib/graphics/selectors/_linear_region.py b/fastplotlib/graphics/selectors/_linear_region.py index 8a8583ae9..10dcfdc3e 100644 --- a/fastplotlib/graphics/selectors/_linear_region.py +++ b/fastplotlib/graphics/selectors/_linear_region.py @@ -341,6 +341,12 @@ def get_selected_data( """ source = self._get_source(graphic) + + if source.data.value is None: + raise ValueError( + "Cannot get selected data. The graphic has no local buffer, `cpu_buffer` is probably `False`." + ) + ixs = self.get_selected_indices(source) if "Line" in source.__class__.__name__: diff --git a/fastplotlib/graphics/selectors/_polygon.py b/fastplotlib/graphics/selectors/_polygon.py index e02c627ac..5a05bc886 100644 --- a/fastplotlib/graphics/selectors/_polygon.py +++ b/fastplotlib/graphics/selectors/_polygon.py @@ -200,6 +200,12 @@ def get_selected_data( view or list of views of the full array, returns empty array if selection is empty """ source = self._get_source(graphic) + + if source.data.value is None: + raise ValueError( + "Cannot get selected data. The graphic has no local buffer, `cpu_buffer` is probably `False`." + ) + ixs = self.get_selected_indices(source) # do not need to check for mode for images, because the selector is bounded by the image shape diff --git a/fastplotlib/graphics/selectors/_rectangle.py b/fastplotlib/graphics/selectors/_rectangle.py index e30165dae..f15f292f8 100644 --- a/fastplotlib/graphics/selectors/_rectangle.py +++ b/fastplotlib/graphics/selectors/_rectangle.py @@ -381,6 +381,12 @@ def get_selected_data( view or list of views of the full array, returns empty array if selection is empty """ source = self._get_source(graphic) + + if source.data.value is None: + raise ValueError( + "Cannot get selected data. The graphic has no local buffer, `cpu_buffer` is probably `False`." + ) + ixs = self.get_selected_indices(source) # do not need to check for mode for images, because the selector is bounded by the image shape diff --git a/fastplotlib/layouts/_figure.py b/fastplotlib/layouts/_figure.py index 013ce847c..f166c18ae 100644 --- a/fastplotlib/layouts/_figure.py +++ b/fastplotlib/layouts/_figure.py @@ -19,7 +19,7 @@ from ._utils import controller_types as valid_controller_types from ._subplot import Subplot from ._engine import GridLayout, WindowLayout, ScreenSpaceCamera -from .. import ImageGraphic +from .. import ImageGraphic, ImageYUVGraphic class Figure: @@ -617,7 +617,7 @@ def show( # flip y-axis if ImageGraphics are present for subplot in self._subplots.ravel(): for g in subplot.graphics: - if isinstance(g, ImageGraphic): + if isinstance(g, (ImageGraphic, ImageYUVGraphic)): if subplot.camera.local.scale_y == 1: # if it's 1 it's likely not been touched manually before show was called subplot.camera.local.scale_y = -1 diff --git a/fastplotlib/layouts/_graphic_methods_mixin.py b/fastplotlib/layouts/_graphic_methods_mixin.py index 1fbf337e2..9eae4dd12 100644 --- a/fastplotlib/layouts/_graphic_methods_mixin.py +++ b/fastplotlib/layouts/_graphic_methods_mixin.py @@ -8,6 +8,8 @@ from ..graphics import * from ..graphics._base import Graphic +import typing +import fastplotlib class GraphicMethodsMixin: @@ -33,11 +35,13 @@ def add_image( cmap: str = "plasma", interpolation: str = "nearest", cmap_interpolation: str = "linear", + colorspace: fastplotlib.utils.enums.ColorspacesRGB = "srgb", + cpu_buffer: bool = True, **kwargs ) -> ImageGraphic: """ - Create an Image Graphic + Create an ImageGraphic Parameters ---------- @@ -61,6 +65,38 @@ def add_image( cmap_interpolation: str, optional, default "linear" colormap interpolation method, one of "nearest" or "linear" + colorspace: one of "srgb", "tex-srgb", "physical", default "srgb" + colorspace in which to interpret the provided data. + + * "srgb": the data represents intensity, rgb, or rgba pixels in the sRGB space. + sRGB is a standard color space designed for consistent representation of colors + across devices like monitors. Most images store colors in this space. + The shader convers sRGB colors to physical in the shader before doing color computations. + + * "tex-srgb": the underlying texture will be of an sRGB format. This means the data + is automatically converted to sRGB when it is sampled. This results in better glTF + compliance (because interpolation in the sampling happens in linear space). + Note that sampling *always* results in the sRGB values, also when not interpreted as color. + Only supported for rgb and rgba data. + + * "physical": the colors are (already) in the physical / linear space, where lighting + calculations can be applied. Shader code that interprets the data as color will use it as-is. + + cpu_buffer: bool, default True + If ``True``, maintains a buffer of system RAM that is sychronized with a corresponding storage buffer + on the GPU. + If ``False``, setting the graphic data will send the new data directly to the GPU, we also + call this "bufferless". This is much faster but lacks the following features: + * you must update the entire data array, i.e. you can perform ``image.data = new_data``, and you + cannot perform partial updates such as ``image.data[indices] = ``. + * RGB arrays of shape [rows, cols, 3] are not supported since wgpu does not have RGB textures, + use RGBA or use `cpu_buffer=True` if you really need RGB instead of RGBA. + * tooltip values for grayscale data are estimated using an inverse transforms on the colormap LUT. + The tooltip values may or may not be accurate for a given colormap and vmin, vmax. If you require + precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. + * vmin, vmax must be explicitly provided if sharing an existing buffer from another ImageGraphic + * ``reset_vmin_vmax()`` is not supported + kwargs: additional keyword arguments passed to :class:`.Graphic` @@ -74,6 +110,8 @@ def add_image( cmap, interpolation, cmap_interpolation, + colorspace, + cpu_buffer, **kwargs ) @@ -172,6 +210,98 @@ def add_image_volume( **kwargs ) + def add_image_yuv( + self, + data: ( + tuple[ + numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.uint8]], + numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.uint8]], + numpy.ndarray[tuple[typing.Any, ...], numpy.dtype[numpy.uint8]], + ] + | fastplotlib.graphics.features._image.TextureYUV + ), + vmin: float = 0, + vmax: float = 255, + interpolation: str = "nearest", + colorspace: fastplotlib.utils.enums.ColorspacesYUV = "yuv420p", + colorrange: fastplotlib.utils.enums.ColorRange = "limited", + **kwargs + ) -> ImageYUVGraphic: + """ + + Create an ImageYUVGraphic. Similar to ImageGraphic but handles data that is in yuv42p or yuv444p colorspace. + + Note that the buffers for YUV Images only exist on the GPU. When setting the image data, the new values are + directly sent to the GPU. + + ``reset_vmin_vmax()`` just sets (vmin, vmax) to (0, 255) + + Parameters + ---------- + data: TupleYUV + tuple of arrays that represent YUV channels. If the colorspace is yuv420p, the U and V array dims + must be 4 times smaller than the Y array dims. + + vmin: float, optional, default 0 + minimum value for color scaling + + vmax: float, optional, default 255 + maximum value for color scaling + + interpolation: str, optional, default "nearest" + interpolation filter, one of "nearest" or "linear" + + colorspace: "yuv42p" | "yuv444p" + colorspace in which to interpret the provided data. + + * "yuv420p": A common video format. The data is represented as 3 planes (y, u, and v). + The y represents intensity, and is at full resolution. The u and v planes are a + quarter of the size. + + * "yuv444p": A lesser common video format. The data is represented as 3 planes + (y, u, and v) similar to yuv420p however the u and v planes are stored + at full resolution. + + colorrange: Literal["full", "limited"] = "limited", + Relevant for yuv colorspaces. Most videos use "limited". + + * "limited": The luma plane (Y) is limited to the range of 16-235 for 8 bits. + The chroma planes (U and V) are limited to the range of 16-240 for 8 bits + * "full": The luma plane and chroma plane use the full range of the storage format. + + See the following links from the FFMPEG documentation for more details: + https://trac.ffmpeg.org/wiki/colorspace + https://ffmpeg.org/doxygen/7.0/pixfmt_8h_source.html#l00609 + + cpu_buffer: bool, default True + If ``True``, maintains a buffer of system RAM that is sychronized with a corresponding storage buffer + on the GPU. + If ``False``, setting the graphic data will send the new data directly to the GPU, we also + call this "bufferless". This is much faster but lacks the following features: + * you must update the entire data array, i.e. you can perform ``image.data = new_data``, and you + cannot perform partial updates such as ``image.data[indices] = ``. + * RGB arrays of shape [rows, cols, 3] are not supported since wgpu does not have RGB textures, + use RGBA or use `cpu_buffer=True` if you really need RGB instead of RGBA. + * tooltip values for grayscale data are estimated using an inverse transforms on the colormap LUT. + The tooltip values may or may not be accurate for a given colormap and vmin, vmax. If you require + precise and reliable tooltip values for grayscale data use `cpu_buffer=True`. + + kwargs: + additional keyword arguments passed to :class:`.Graphic` + + + """ + return self._create_graphic( + ImageYUVGraphic, + data, + vmin, + vmax, + interpolation, + colorspace, + colorrange, + **kwargs + ) + def add_line_collection( self, data: Union[numpy.ndarray, List[numpy.ndarray]], From 16216fbe8a20ba3319c70ac79880b9e57f67a7c4 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 19:33:58 -0400 Subject: [PATCH 13/18] add enum to top level namespace --- fastplotlib/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fastplotlib/__init__.py b/fastplotlib/__init__.py index d975f4d0a..c4626a041 100644 --- a/fastplotlib/__init__.py +++ b/fastplotlib/__init__.py @@ -6,6 +6,7 @@ from .utils import loop # noqa from .utils import ( config, + enums, enumerate_adapters, select_adapter, print_wgpu_report, From f77593a3304ff695108c076af613b83af2d838ab Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 19:34:20 -0400 Subject: [PATCH 14/18] update script to produce add graphics mixin --- scripts/generate_add_graphic_methods.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/generate_add_graphic_methods.py b/scripts/generate_add_graphic_methods.py index 865eab27f..c5a526e93 100644 --- a/scripts/generate_add_graphic_methods.py +++ b/scripts/generate_add_graphic_methods.py @@ -35,7 +35,10 @@ def generate_add_graphics_methods(): f.write("import numpy\n\n") f.write("import pygfx\n\n") f.write("from ..graphics import *\n") - f.write("from ..graphics._base import Graphic\n\n") + f.write("from ..graphics._base import Graphic\n") + f.write("from ..utils import enums\n") + f.write("import typing\n") + f.write("import fastplotlib\n\n") f.write("\nclass GraphicMethodsMixin:\n") @@ -52,11 +55,14 @@ def generate_add_graphics_methods(): f.write(" self.add_graphic(graphic, center=center)\n\n") f.write(" return graphic\n\n") + # from https://stackoverflow.com/a/1176023 + camel_to_snake = re.compile(r"(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") + for m in modules: cls = m cls_name = cls.__name__.replace("Graphic", "") - # from https://stackoverflow.com/a/1176023 - method_name = re.sub(r"(? Date: Thu, 16 Apr 2026 19:34:36 -0400 Subject: [PATCH 15/18] add yuv example --- examples/image/image_yuv.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/image/image_yuv.py b/examples/image/image_yuv.py index a4790a32d..dfb7cad47 100644 --- a/examples/image/image_yuv.py +++ b/examples/image/image_yuv.py @@ -10,6 +10,9 @@ YUV is also called YCbCr for digital images. For more info: https://en.wikipedia.org/wiki/Y%E2%80%B2UV + +You can see the slight differences between yuv420 and yuv444 if you zoom into parts of the image where colors change +rapidly over space, such as the astronaut's patch. """ # test_example = true @@ -25,16 +28,24 @@ img_yuv = rgb2ycbcr(img).astype(np.uint8) y = img_yuv[..., 0] -u = img_yuv[::2, ::2, 1] -v = img_yuv[::2, ::2, 2] +u = img_yuv[..., 1] +v = img_yuv[..., 2] -figure = fpl.Figure(size=(700, 560)) +figure = fpl.Figure( + shape=(1, 2), names=["yuv420p", "yuv444p"], controller_ids="sync", size=(700, 400) +) -# plot the image data -image = figure[0, 0].add_image_yuv( - data=(y, u, v), colorspace="yuv420p", name="yuv image" +image1 = figure[0, 0].add_image_yuv( + data=(y, u[::2, ::2], v[::2, ::2]), colorspace="yuv420p" ) +image2 = figure[0, 1].add_image_yuv(data=(y, u, v), colorspace="yuv444p") + +cursor = fpl.Cursor() + +for subplot in figure: + cursor.add_subplot(subplot) + figure.show() From 8b14f40c8eedaa671c950fb9acc8cbb7ffe240e7 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 19:41:53 -0400 Subject: [PATCH 16/18] update ndimage with yuv stuff --- fastplotlib/widgets/nd_widget/_nd_image.py | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/fastplotlib/widgets/nd_widget/_nd_image.py b/fastplotlib/widgets/nd_widget/_nd_image.py index 5c8942363..ee027c49c 100644 --- a/fastplotlib/widgets/nd_widget/_nd_image.py +++ b/fastplotlib/widgets/nd_widget/_nd_image.py @@ -5,8 +5,8 @@ from numpy.typing import ArrayLike from ...layouts import Subplot -from ...utils import subsample_array, ARRAY_LIKE_ATTRS, ArrayProtocol -from ...graphics import ImageGraphic, ImageVolumeGraphic +from ...utils import subsample_array, ARRAY_LIKE_ATTRS, ArrayProtocol, enums +from ...graphics import ImageGraphic, ImageYUVGraphic, ImageVolumeGraphic from ...tools import HistogramLUTTool from ._base import ( NDProcessor, @@ -385,7 +385,7 @@ def __init__( self._colorspace = colorspace self._colorrange = colorrange - self._graphic: ImageGraphic | None = None + self._graphic: ImageGraphic | ImageYUVGraphic | None = None self._histogram_widget: HistogramLUTTool | None = None # create a graphic @@ -399,7 +399,7 @@ def processor(self) -> NDImageProcessor: @property def graphic( self, - ) -> ImageGraphic | ImageVolumeGraphic: + ) -> ImageGraphic | ImageYUVGraphic | ImageVolumeGraphic: """Underlying Graphic object used to display the current data slice""" return self._graphic @@ -412,15 +412,23 @@ def _create_graphic(self): # no graphic if data is None, useful for initializing in null states when we want to set data later return - # determine if we need a 2d image or 3d volume - # remove RGB spatial dim, ex: if we have an RGBA image of shape [512, 512, 4] we want to interpet this as - # 2D for images - # [30, 512, 512, 4] with an rgb dim is an RGBA volume which is also supported - match len(self.processor.spatial_dims) - int(bool(self.processor.rgb_dim)): - case 2: - cls = ImageGraphic - case 3: - cls = ImageVolumeGraphic + kwargs = { + "colorspace": self._colorspace, + } + + if self._colorspace in enums.ColorspacesYUV: + cls = ImageYUVGraphic + kwargs["colorrange"] = self._colorrange + else: + # determine if we need a 2d image or 3d volume + # remove RGB spatial dim, ex: if we have an RGBA image of shape [512, 512, 4] we want to interpet this as + # 2D for images + # [30, 512, 512, 4] with an rgb dim is an RGBA volume which is also supported + match len(self.processor.spatial_dims) - int(bool(self.processor.rgb_dim)): + case 2: + cls = ImageGraphic + case 3: + cls = ImageVolumeGraphic # get the data slice for this index # this will only have the dims specified by ``spatial_dims`` @@ -430,9 +438,8 @@ def _create_graphic(self): # create the new graphic new_graphic = cls( data_slice, - cpu_buffer=False, # faster, we usually don't need a cpu buffer for NDWidget use cases - colorspace=self._colorspace, - colorrange=self._colorrange, + # cpu_buffer=False, # faster, we usually don't need a cpu buffer for NDWidget use cases + **kwargs, ) old_graphic = self._graphic @@ -489,7 +496,7 @@ def _reset_histogram(self): def _reset_camera(self): # set camera to a nice position based on whether it's a 2D ImageGraphic or 3D ImageVolumeGraphic - if isinstance(self._graphic, ImageGraphic): + if isinstance(self._graphic, (ImageGraphic, ImageYUVGraphic)): # set camera orthogonal to the xy plane, flip y axis self._subplot.camera.set_state( { From 776ea0593a6ecaa41892e4bc97f9d47589d95663 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 20:24:47 -0400 Subject: [PATCH 17/18] fixes --- fastplotlib/graphics/features/_image.py | 10 ++++------ fastplotlib/graphics/image.py | 14 +++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/fastplotlib/graphics/features/_image.py b/fastplotlib/graphics/features/_image.py index fba629ebe..a2c6f1183 100644 --- a/fastplotlib/graphics/features/_image.py +++ b/fastplotlib/graphics/features/_image.py @@ -57,7 +57,7 @@ def __init__( if cpu_buffer: # create a local buffer - self._value = np.zeros(data.shape, dtype=data.dtype) + self._value = np.empty(data.shape, dtype=data.dtype) self.value[:] = data[:] else: self._value = None @@ -99,18 +99,16 @@ def __init__( # iterate through each chunk of passed `data` # create a pygfx.Texture from this chunk for _, buffer_index, slicer in self: - chunk = data[slicer] - if cpu_buffer: # texture gets the data directly texture = pygfx.Texture( - chunk, + self.value[slicer], dim=2, colorspace=colorspace, ) else: # we only supply the size - w, h = chunk.shape[1], chunk.shape[0] + w, h = data[slicer].shape[1], data[slicer].shape[0] texture = pygfx.Texture( size=(w, h, 1), @@ -121,7 +119,7 @@ def __init__( ) # send the initial data - texture.send_data((0, 0, 0), chunk) + texture.send_data((0, 0, 0), data[slicer]) self.buffer[buffer_index] = texture diff --git a/fastplotlib/graphics/image.py b/fastplotlib/graphics/image.py index aff0927f5..3d17d44a2 100644 --- a/fastplotlib/graphics/image.py +++ b/fastplotlib/graphics/image.py @@ -447,7 +447,6 @@ def __init__( if isinstance(data, TextureArray): # share buffer self._data = data - data = self._data.value else: # create new texture array to manage buffer # texture array that manages the multiple textures on the GPU that represent this image @@ -455,17 +454,17 @@ def __init__( data, colorspace=colorspace, cpu_buffer=cpu_buffer ) - if (vmin is None) or (vmax is None) and data is not None: + if (vmin is None) or (vmax is None): + if self.data.value is None: + raise ValueError( + "must provide vmin, vmax if sharing a buffer that does not exist locally" + ) + _vmin, _vmax = quick_min_max(self.data.value) if vmin is None: vmin = _vmin if vmax is None: vmax = _vmax - else: - # this is a shared buffer and we don't have access to the actual data from here - raise ValueError( - "must provide vmin, vmax if sharing a buffer that does not exist locally" - ) # other graphic features self._vmin = ImageVmin(vmin) @@ -583,6 +582,7 @@ def data(self, data): self._data[:] = data + @property def colorspace(self) -> ColorspacesRGB: """The image's colorspace""" From 64938c50b870b59f2aa73cfedc7e6e1b317ad924 Mon Sep 17 00:00:00 2001 From: kushalkolar Date: Thu, 16 Apr 2026 20:38:38 -0400 Subject: [PATCH 18/18] yuv video working well with NDWidget --- fastplotlib/widgets/nd_widget/_nd_image.py | 2 +- fastplotlib/widgets/nd_widget/_ndw_subplot.py | 29 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/fastplotlib/widgets/nd_widget/_nd_image.py b/fastplotlib/widgets/nd_widget/_nd_image.py index ee027c49c..6463eba88 100644 --- a/fastplotlib/widgets/nd_widget/_nd_image.py +++ b/fastplotlib/widgets/nd_widget/_nd_image.py @@ -416,7 +416,7 @@ def _create_graphic(self): "colorspace": self._colorspace, } - if self._colorspace in enums.ColorspacesYUV: + if self._colorspace in {cs.value for cs in enums.ColorspacesYUV}: cls = ImageYUVGraphic kwargs["colorrange"] = self._colorrange else: diff --git a/fastplotlib/widgets/nd_widget/_ndw_subplot.py b/fastplotlib/widgets/nd_widget/_ndw_subplot.py index 5fbd8d8f6..3c655d662 100644 --- a/fastplotlib/widgets/nd_widget/_ndw_subplot.py +++ b/fastplotlib/widgets/nd_widget/_ndw_subplot.py @@ -12,9 +12,10 @@ VectorsGraphic, ) from ...layouts import Subplot -from ...utils import ArrayProtocol -from . import NDImage, NDPositions, NDVectors -from ._base import NDGraphic, WindowFuncCallable +from ...utils import ArrayProtocol, enums +from . import NDImageProcessor, NDImage, NDPositions, NDVectors +from ._video import VideoProcessor +from ._base import NDProcessor, NDGraphic, WindowFuncCallable class NDWSubplot: @@ -86,6 +87,28 @@ def add_nd_image( self._nd_graphics.append(nd) return nd + def add_video( + self, + data: ArrayProtocol | None, + dims: Sequence[str], + spatial_dims: tuple[str, str] | tuple[str, str, str], + rgb_dim: str | None = None, + colorspace: enums.ColorspacesYUV | enums.ColorspacesRGB = "yuv420p", + colorrange: enums.ColorRange = "limited", + processor_type: NDImageProcessor = VideoProcessor, + **kwargs, + ): + return self.add_nd_image( + data=data, + dims=dims, + spatial_dims=spatial_dims, + rgb_dim=rgb_dim, + colorspace=colorspace, + colorrange=colorrange, + processor_type=processor_type, + **kwargs, + ) + def add_nd_vectors( self, data: ArrayProtocol | None,