From c097016fd2a0489fdd77387bb8f087316af1dfc0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 09:38:25 -0400 Subject: [PATCH 001/113] Add new `ui._flows` module This begins the removal of data processing / analysis methods from the chart widget and instead moving them to our new `Flow` API (in the new module introduce here) and delegating the old chart methods to the respective internal flow. Most importantly is no longer storing the "last read" of an array from shm in an internal chart table (was `._arrays`) and instead the `ShmArray` instance is passed as input and stored in the `Flow` instance. This greatly simplifies lookup logic such that the display loop now doesn't have to worry about reading shm, it can be done by internal graphics logic as desired. Generally speaking, all previous `._arrays`/`._graphics` lookups are now delegated to the entries in the chart's `._flows` table. The new `Flow` methods are generally better factored and provide more detailed output regarding data-stream <-> graphics inter-relations for the future purpose of allowing much more efficient update calls in the display loop as well as supporting low latency interaction UX. The concept here is that we're introducing an intermediary layer that ties together graphics and real-time data flows such that widget code is oriented around plot layout and the flow apis are oriented around real-time low latency updates and providing an efficient high level metric layer for the UX. The summary api transition is something like: - `update_graphics_from_array()` -> `.update_graphics_from_flow()` - `.bars_range()` -> `Flow.datums_range()` - `.bars_range()` -> `Flow.datums_range()` --- piker/ui/_chart.py | 277 +++++++++-------------------------------- piker/ui/_flows.py | 303 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+), 220 deletions(-) create mode 100644 piker/ui/_flows.py diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index a3a971648..8aa100911 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -34,9 +34,7 @@ QVBoxLayout, QSplitter, ) -import msgspec import numpy as np -# from pydantic import BaseModel import pyqtgraph as pg import trio @@ -49,6 +47,7 @@ Cursor, ContentsLabel, ) +from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems from ._curve import FastAppendCurve @@ -60,15 +59,12 @@ ) from ..data.feed import Feed from ..data._source import Symbol -from ..data._sharedmem import ( - ShmArray, - # _Token, -) from ..log import get_logger from ._interaction import ChartView from ._forms import FieldsForm from .._profile import pg_profile_enabled, ms_slower_then from ._overlay import PlotItemOverlay +from ._flows import Flow if TYPE_CHECKING: from ._display import DisplayState @@ -419,7 +415,7 @@ def plot_ohlc_main( self, symbol: Symbol, - array: np.ndarray, + shm: ShmArray, sidepane: FieldsForm, style: str = 'bar', @@ -444,7 +440,7 @@ def plot_ohlc_main( self.chart = self.add_plot( name=symbol.key, - array=array, + shm=shm, style=style, _is_main=True, @@ -472,7 +468,7 @@ def add_plot( self, name: str, - array: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, style: str = 'line', @@ -516,7 +512,6 @@ def add_plot( name=name, data_key=array_key or name, - array=array, parent=qframe, linkedsplits=self, axisItems=axes, @@ -580,7 +575,7 @@ def add_plot( graphics, data_key = cpw.draw_ohlc( name, - array, + shm, array_key=array_key ) self.cursor.contents_labels.add_label( @@ -594,7 +589,7 @@ def add_plot( add_label = True graphics, data_key = cpw.draw_curve( name, - array, + shm, array_key=array_key, color='default_light', ) @@ -603,7 +598,7 @@ def add_plot( add_label = True graphics, data_key = cpw.draw_curve( name, - array, + shm, array_key=array_key, step_mode=True, color='davies', @@ -691,7 +686,6 @@ def __init__( # the "data view" we generate graphics from name: str, - array: np.ndarray, data_key: str, linkedsplits: LinkedSplits, @@ -744,14 +738,6 @@ def __init__( self._max_l1_line_len: float = 0 # self.setViewportMargins(0, 0, 0, 0) - # self._ohlc = array # readonly view of ohlc data - - # TODO: move to Aggr above XD - # readonly view of data arrays - self._arrays = { - self.data_key: array, - } - self._graphics = {} # registry of underlying graphics # registry of overlay curve names self._flows: dict[str, Flow] = {} @@ -767,7 +753,6 @@ def __init__( # show background grid self.showGrid(x=False, y=True, alpha=0.3) - self.default_view() self.cv.enable_auto_yrange() self.pi_overlay: PlotItemOverlay = PlotItemOverlay(self.plotItem) @@ -816,14 +801,8 @@ def bars_range(self) -> tuple[int, int, int, int]: Return a range tuple for the bars present in view. ''' - l, r = self.view_range() - array = self._arrays[self.name] - start, stop = self._xrange = ( - array[0]['index'], - array[-1]['index'], - ) - lbar = max(l, start) - rbar = min(r, stop) + main_flow = self._flows[self.name] + ifirst, l, lbar, rbar, r, ilast = main_flow.datums_range() return l, lbar, rbar, r def curve_width_pxs( @@ -877,40 +856,51 @@ def marker_right_points( def default_view( self, - steps_on_screen: Optional[int] = None + bars_from_y: int = 5000, ) -> None: ''' Set the view box to the "default" startup view of the scene. ''' - try: - index = self._arrays[self.name]['index'] - except IndexError: - log.warning(f'array for {self.name} not loaded yet?') + flow = self._flows.get(self.name) + if not flow: + log.warning(f'`Flow` for {self.name} not loaded yet?') return + index = flow.shm.array['index'] xfirst, xlast = index[0], index[-1] l, lbar, rbar, r = self.bars_range() - - marker_pos, l1_len = self.pre_l1_xs() - end = xlast + l1_len + 1 + view = self.view if ( rbar < 0 or l < xfirst + or l < 0 or (rbar - lbar) < 6 ): - # set fixed bars count on screen that approx includes as + # TODO: set fixed bars count on screen that approx includes as # many bars as possible before a downsample line is shown. - begin = xlast - round(6116 / 6) + begin = xlast - bars_from_y + view.setXRange( + min=begin, + max=xlast, + padding=0, + ) + # re-get range + l, lbar, rbar, r = self.bars_range() - else: - begin = end - (r - l) + # we get the L1 spread label "length" in view coords + # terms now that we've scaled either by user control + # or to the default set of bars as per the immediate block + # above. + marker_pos, l1_len = self.pre_l1_xs() + end = xlast + l1_len + 1 + begin = end - (r - l) # for debugging # print( - # f'bars range: {brange}\n' + # # f'bars range: {brange}\n' # f'xlast: {xlast}\n' # f'marker pos: {marker_pos}\n' # f'l1 len: {l1_len}\n' @@ -922,14 +912,13 @@ def default_view( if self._static_yrange == 'axis': self._static_yrange = None - view = self.view view.setXRange( min=begin, max=end, padding=0, ) - view._set_yrange() self.view.maybe_downsample_graphics() + view._set_yrange() try: self.linked.graphics_cycle() except IndexError: @@ -960,7 +949,7 @@ def increment_view( def draw_ohlc( self, name: str, - data: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, @@ -980,19 +969,21 @@ def draw_ohlc( # the np array buffer to be drawn on next render cycle self.plotItem.addItem(graphics) - # draw after to allow self.scene() to work... - graphics.draw_from_data(data) - data_key = array_key or name - self._graphics[data_key] = graphics self._flows[data_key] = Flow( name=name, plot=self.plotItem, + _shm=shm, is_ohlc=True, graphics=graphics, ) + # TODO: i think we can eventually remove this if + # we write the ``Flow.update_graphics()`` method right? + # draw after to allow self.scene() to work... + graphics.draw_from_data(shm.array) + self._add_sticky(name, bg_color='davies') return graphics, data_key @@ -1058,7 +1049,7 @@ def draw_curve( self, name: str, - data: np.ndarray, + shm: ShmArray, array_key: Optional[str] = None, overlay: bool = False, @@ -1071,7 +1062,7 @@ def draw_curve( ) -> (pg.PlotDataItem, str): ''' Draw a "curve" (line plot graphics) for the provided data in - the input array ``data``. + the input shm array ``shm``. ''' color = color or self.pen_color or 'default_light' @@ -1082,6 +1073,7 @@ def draw_curve( data_key = array_key or name # yah, we wrote our own B) + data = shm.array curve = FastAppendCurve( y=data[data_key], x=data['index'], @@ -1105,16 +1097,14 @@ def draw_curve( # and is disastrous for performance. # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) - # register curve graphics and backing array for name - self._graphics[name] = curve - self._arrays[data_key] = data - pi = pi or self.plotItem self._flows[data_key] = Flow( name=name, plot=pi, + _shm=shm, is_ohlc=False, + # register curve graphics with this flow graphics=curve, ) @@ -1175,16 +1165,11 @@ def _add_sticky( ) return last - def update_graphics_from_array( + def update_graphics_from_flow( self, graphics_name: str, - - array: Optional[np.ndarray] = None, array_key: Optional[str] = None, - use_vr: bool = True, - render: bool = True, - **kwargs, ) -> pg.GraphicsObject: @@ -1192,63 +1177,11 @@ def update_graphics_from_array( Update the named internal graphics from ``array``. ''' - if array is not None: - assert len(array) - - data_key = array_key or graphics_name - if graphics_name not in self._flows: - data_key = self.name - - if array is not None: - # write array to internal graphics table - self._arrays[data_key] = array - else: - array = self._arrays[data_key] - - # array key and graphics "name" might be different.. - graphics = self._graphics[graphics_name] - - # compute "in-view" indices - l, lbar, rbar, r = self.bars_range() - indexes = array['index'] - ifirst = indexes[0] - ilast = indexes[-1] - - lbar_i = max(l, ifirst) - ifirst - rbar_i = min(r, ilast) - ifirst - - # TODO: we could do it this way as well no? - # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] - in_view = array[lbar_i: rbar_i + 1] - - if ( - not in_view.size - or not render - ): - return graphics - - if isinstance(graphics, BarItems): - graphics.update_from_array( - array, - in_view, - view_range=(lbar_i, rbar_i) if use_vr else None, - - **kwargs, - ) - - else: - graphics.update_from_array( - x=array['index'], - y=array[data_key], - - x_iv=in_view['index'], - y_iv=in_view[data_key], - view_range=(lbar_i, rbar_i) if use_vr else None, - - **kwargs - ) - - return graphics + flow = self._flows[array_key or graphics_name] + return flow.update_graphics( + array_key=array_key, + **kwargs, + ) # def _label_h(self, yhigh: float, ylow: float) -> float: # # compute contents label "height" in view terms @@ -1295,7 +1228,7 @@ def get_index(self, time: float) -> int: # TODO: this should go onto some sort of # data-view thinger..right? - ohlc = self._shm.array + ohlc = self._flows[self.name].shm.array # XXX: not sure why the time is so off here # looks like we're gonna have to do some fixing.. @@ -1341,9 +1274,6 @@ def maxmin( delayed=True, ) - l, lbar, rbar, r = bars_range or self.bars_range() - profiler(f'{self.name} got bars range') - # TODO: here we should instead look up the ``Flow.shm.array`` # and read directly from shm to avoid copying to memory first # and then reading it again here. @@ -1356,6 +1286,9 @@ def maxmin( res = 0, 0 else: + first, l, lbar, rbar, r, last = bars_range or flow.datums_range() + profiler(f'{self.name} got bars range') + key = round(lbar), round(rbar) res = flow.maxmin(*key) profiler(f'yrange mxmn: {key} -> {res}') @@ -1366,99 +1299,3 @@ def maxmin( res = 0, 0 return res - - -# class FlowsTable(pydantic.BaseModel): -# ''' -# Data-AGGRegate: high level API onto multiple (categorized) -# ``Flow``s with high level processing routines for -# multi-graphics computations and display. - -# ''' -# flows: dict[str, np.ndarray] = {} - - -class Flow(msgspec.Struct): # , frozen=True): - ''' - (FinancialSignal-)Flow compound type which wraps a real-time - graphics (curve) and its backing data stream together for high level - access and control. - - The intention is for this type to eventually be capable of shm-passing - of incrementally updated graphics stream data between actors. - - ''' - name: str - plot: pg.PlotItem - is_ohlc: bool = False - graphics: pg.GraphicsObject - - # TODO: hackery to be able to set a shm later - # but whilst also allowing this type to hashable, - # likely will require serializable token that is used to attach - # to the underlying shm ref after startup? - _shm: Optional[ShmArray] = None # currently, may be filled in "later" - - # cache of y-range values per x-range input. - _mxmns: dict[tuple[int, int], tuple[float, float]] = {} - - @property - def shm(self) -> ShmArray: - return self._shm - - @shm.setter - def shm(self, shm: ShmArray) -> ShmArray: - self._shm = shm - - def maxmin( - self, - lbar, - rbar, - - ) -> tuple[float, float]: - ''' - Compute the cached max and min y-range values for a given - x-range determined by ``lbar`` and ``rbar``. - - ''' - rkey = (lbar, rbar) - cached_result = self._mxmns.get(rkey) - if cached_result: - return cached_result - - shm = self.shm - if shm is None: - mxmn = None - - else: # new block for profiling?.. - arr = shm.array - - # build relative indexes into shm array - # TODO: should we just add/use a method - # on the shm to do this? - ifirst = arr[0]['index'] - slice_view = arr[ - lbar - ifirst: - (rbar - ifirst) + 1 - ] - - if not slice_view.size: - mxmn = None - - else: - if self.is_ohlc: - ylow = np.min(slice_view['low']) - yhigh = np.max(slice_view['high']) - - else: - view = slice_view[self.name] - ylow = np.min(view) - yhigh = np.max(view) - - mxmn = ylow, yhigh - - if mxmn is not None: - # cache new mxmn result - self._mxmns[rkey] = mxmn - - return mxmn diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py new file mode 100644 index 000000000..a9eb6a4f3 --- /dev/null +++ b/piker/ui/_flows.py @@ -0,0 +1,303 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for pikers) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +''' +High level streaming graphics primitives. + +This is an intermediate layer which associates real-time low latency +graphics primitives with underlying FSP related data structures for fast +incremental update. + +''' +from typing import ( + Optional, + Callable, +) + +import msgspec +import numpy as np +import pyqtgraph as pg +from PyQt5.QtGui import QPainterPath + +from ..data._sharedmem import ( + ShmArray, + # attach_shm_array +) +from ._ohlc import BarItems + + +# class FlowsTable(msgspec.Struct): +# ''' +# Data-AGGRegate: high level API onto multiple (categorized) +# ``Flow``s with high level processing routines for +# multi-graphics computations and display. + +# ''' +# flows: dict[str, np.ndarray] = {} + + +class Flow(msgspec.Struct): # , frozen=True): + ''' + (FinancialSignal-)Flow compound type which wraps a real-time + graphics (curve) and its backing data stream together for high level + access and control. + + The intention is for this type to eventually be capable of shm-passing + of incrementally updated graphics stream data between actors. + + ''' + name: str + plot: pg.PlotItem + is_ohlc: bool = False + render: bool = True # toggle for display loop + + graphics: pg.GraphicsObject + + # TODO: hackery to be able to set a shm later + # but whilst also allowing this type to hashable, + # likely will require serializable token that is used to attach + # to the underlying shm ref after startup? + _shm: Optional[ShmArray] = None # currently, may be filled in "later" + + # last read from shm (usually due to an update call) + _last_read: Optional[np.ndarray] = None + + # cache of y-range values per x-range input. + _mxmns: dict[tuple[int, int], tuple[float, float]] = {} + + @property + def shm(self) -> ShmArray: + return self._shm + + # TODO: remove this and only allow setting through + # private ``._shm`` attr? + @shm.setter + def shm(self, shm: ShmArray) -> ShmArray: + print(f'{self.name} DO NOT SET SHM THIS WAY!?') + self._shm = shm + + def maxmin( + self, + lbar, + rbar, + + ) -> tuple[float, float]: + ''' + Compute the cached max and min y-range values for a given + x-range determined by ``lbar`` and ``rbar``. + + ''' + rkey = (lbar, rbar) + cached_result = self._mxmns.get(rkey) + if cached_result: + return cached_result + + shm = self.shm + if shm is None: + mxmn = None + + else: # new block for profiling?.. + arr = shm.array + + # build relative indexes into shm array + # TODO: should we just add/use a method + # on the shm to do this? + ifirst = arr[0]['index'] + slice_view = arr[ + lbar - ifirst: + (rbar - ifirst) + 1 + ] + + if not slice_view.size: + mxmn = None + + else: + if self.is_ohlc: + ylow = np.min(slice_view['low']) + yhigh = np.max(slice_view['high']) + + else: + view = slice_view[self.name] + ylow = np.min(view) + yhigh = np.max(view) + + mxmn = ylow, yhigh + + if mxmn is not None: + # cache new mxmn result + self._mxmns[rkey] = mxmn + + return mxmn + + def view_range(self) -> tuple[int, int]: + ''' + Return the indexes in view for the associated + plot displaying this flow's data. + + ''' + vr = self.plot.viewRect() + return int(vr.left()), int(vr.right()) + + def datums_range(self) -> tuple[ + int, int, int, int, int, int + ]: + ''' + Return a range tuple for the datums present in view. + + ''' + l, r = self.view_range() + + # TODO: avoid this and have shm passed + # in earlier. + if self.shm is None: + # haven't initialized the flow yet + return (0, l, 0, 0, r, 0) + + array = self.shm.array + index = array['index'] + start = index[0] + end = index[-1] + lbar = max(l, start) + rbar = min(r, end) + return ( + start, l, lbar, rbar, r, end, + ) + + def read(self) -> tuple[ + int, int, np.ndarray, + int, int, np.ndarray, + ]: + array = self.shm.array + indexes = array['index'] + ifirst = indexes[0] + ilast = indexes[-1] + + ifirst, l, lbar, rbar, r, ilast = self.datums_range() + + # get read-relative indices adjusting + # for master shm index. + lbar_i = max(l, ifirst) - ifirst + rbar_i = min(r, ilast) - ifirst + + # TODO: we could do it this way as well no? + # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] + in_view = array[lbar_i: rbar_i + 1] + + return ( + # abs indices + full data set + ifirst, ilast, array, + + # relative indices + in view datums + lbar_i, rbar_i, in_view, + ) + + def update_graphics( + self, + use_vr: bool = True, + render: bool = True, + array_key: Optional[str] = None, + + **kwargs, + + ) -> pg.GraphicsObject: + ''' + Read latest datums from shm and render to (incrementally) + render to graphics. + + ''' + # shm read and slice to view + xfirst, xlast, array, ivl, ivr, in_view = self.read() + + if ( + not in_view.size + or not render + ): + return self.graphics + + array_key = array_key or self.name + + graphics = self.graphics + if isinstance(graphics, BarItems): + graphics.update_from_array( + array, + in_view, + view_range=(ivl, ivr) if use_vr else None, + + **kwargs, + ) + + else: + graphics.update_from_array( + x=array['index'], + y=array[array_key], + + x_iv=in_view['index'], + y_iv=in_view[array_key], + view_range=(ivl, ivr) if use_vr else None, + + **kwargs + ) + + return graphics + + # @classmethod + # def from_token( + # cls, + # shm_token: tuple[ + # str, + # str, + # tuple[str, str], + # ], + + # ) -> PathRenderer: + + # shm = attach_shm_array(token) + # return cls(shm) + + +class PathRenderer(msgspec.Struct): + + # output graphics rendering + path: Optional[QPainterPath] = None + + last_read_src_array: np.ndarray + # called on input data but before + prerender_fn: Callable[ShmArray, np.ndarray] + + def diff( + self, + ) -> dict[str, np.ndarray]: + ... + + def update(self) -> QPainterPath: + ''' + Incrementally update the internal path graphics from + updates in shm data and deliver the new (sub)-path + generated. + + ''' + ... + + + def render( + self, + + ) -> list[QPainterPath]: + ''' + Render the current graphics path(s) + + ''' + ... From 599c77ff845e446a1490192250afccfd9c038606 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 10:04:18 -0400 Subject: [PATCH 002/113] Port ui components to use flows, drop all late assignments of shm --- piker/ui/_axes.py | 5 +++-- piker/ui/_cursor.py | 13 +++++++------ piker/ui/_display.py | 33 ++++++++++++++++----------------- piker/ui/_editors.py | 2 +- piker/ui/_fsp.py | 38 ++++++++++---------------------------- 5 files changed, 37 insertions(+), 54 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 93ac7af7b..7ba520555 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -223,8 +223,9 @@ def _indexes_to_timestrs( ) -> list[str]: chart = self.linkedsplits.chart - bars = chart._arrays[chart.name] - shm = self.linkedsplits.chart._shm + flow = chart._flows[chart.name] + shm = flow.shm + bars = shm.array first = shm._first.value bars_len = len(bars) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index a34c15c1f..43207b9f6 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -254,13 +254,13 @@ def __init__( def update_labels( self, index: int, - # array_name: str, ) -> None: - # for name, (label, update) in self._labels.items(): for chart, name, label, update in self._labels: - array = chart._arrays[name] + flow = chart._flows[name] + array = flow.shm.array + if not ( index >= 0 and index < array[-1]['index'] @@ -269,8 +269,6 @@ def update_labels( print('WTF out of range?') continue - # array = chart._arrays[name] - # call provided update func with data point try: label.show() @@ -472,9 +470,12 @@ def add_curve_cursor( ) -> LineDot: # if this plot contains curves add line dot "cursors" to denote # the current sample under the mouse + main_flow = plot._flows[plot.name] + # read out last index + i = main_flow.shm.array[-1]['index'] cursor = LineDot( curve, - index=plot._arrays[plot.name][-1]['index'], + index=i, plot=plot ) plot.addItem(cursor) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 927ce5df6..4b695b04a 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -408,9 +408,8 @@ def graphics_update_cycle( ): # TODO: make it so this doesn't have to be called # once the $vlm is up? - vlm_chart.update_graphics_from_array( + vlm_chart.update_graphics_from_flow( 'volume', - array, # UGGGh, see ``maxmin()`` impl in `._fsp` for # the overlayed plotitems... we need a better @@ -436,6 +435,11 @@ def graphics_update_cycle( vars['last_mx_vlm'] = mx_vlm_in_view for curve_name, flow in vlm_chart._flows.items(): + + if not flow.render: + print(f'skipping flow {curve_name}?') + continue + update_fsp_chart( vlm_chart, flow, @@ -500,9 +504,8 @@ def graphics_update_cycle( or i_diff > 0 or trigger_all ): - chart.update_graphics_from_array( + chart.update_graphics_from_flow( chart.name, - array, ) # iterate in FIFO order per tick-frame @@ -515,8 +518,9 @@ def graphics_update_cycle( # tick frames to determine the y-range for chart # auto-scaling. # TODO: we need a streaming minmax algo here, see def above. - mx = max(price + tick_margin, mx) - mn = min(price - tick_margin, mn) + if liv: + mx = max(price + tick_margin, mx) + mn = min(price - tick_margin, mn) if typ in clear_types: @@ -539,9 +543,8 @@ def graphics_update_cycle( if wap_in_history: # update vwap overlay line - chart.update_graphics_from_array( + chart.update_graphics_from_flow( 'bar_wap', - array, ) # L1 book label-line updates @@ -557,7 +560,7 @@ def graphics_update_cycle( if ( label is not None - # and liv + and liv ): label.update_fields( {'level': price, 'size': size} @@ -571,7 +574,7 @@ def graphics_update_cycle( typ in _tick_groups['asks'] # TODO: instead we could check if the price is in the # y-view-range? - # and liv + and liv ): l1.ask_label.update_fields({'level': price, 'size': size}) @@ -579,7 +582,7 @@ def graphics_update_cycle( typ in _tick_groups['bids'] # TODO: instead we could check if the price is in the # y-view-range? - # and liv + and liv ): l1.bid_label.update_fields({'level': price, 'size': size}) @@ -692,9 +695,10 @@ async def display_symbol_data( # create main OHLC chart chart = linked.plot_ohlc_main( symbol, - bars, + ohlcv, sidepane=pp_pane, ) + chart.default_view() chart._feeds[symbol.key] = feed chart.setFocus() @@ -714,10 +718,6 @@ async def display_symbol_data( # size view to data once at outset chart.cv._set_yrange() - # TODO: a data view api that makes this less shit - chart._shm = ohlcv - chart._flows[chart.data_key].shm = ohlcv - # NOTE: we must immediately tell Qt to show the OHLC chart # to avoid a race where the subplots get added/shown to # the linked set *before* the main price chart! @@ -780,6 +780,5 @@ async def display_symbol_data( sbar._status_groups[loading_sym_key][1]() # let the app run.. bby - chart.default_view() # linked.graphics_cycle() await trio.sleep_forever() diff --git a/piker/ui/_editors.py b/piker/ui/_editors.py index 9a99d2f77..03fd208ea 100644 --- a/piker/ui/_editors.py +++ b/piker/ui/_editors.py @@ -343,7 +343,7 @@ def set_pos( nbars = ixmx - ixmn + 1 chart = self._chart - data = chart._arrays[chart.name][ixmn:ixmx] + data = chart._flows[chart.name].shm.array[ixmn:ixmx] if len(data): std = data['close'].std() diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 527633757..9aa10fb3c 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -93,9 +93,8 @@ def update_fsp_chart( # update graphics # NOTE: this does a length check internally which allows it # staying above the last row check below.. - chart.update_graphics_from_array( + chart.update_graphics_from_flow( graphics_name, - array, array_key=array_key or graphics_name, ) @@ -106,9 +105,6 @@ def update_fsp_chart( # read from last calculated value and update any label last_val_sticky = chart._ysticks.get(graphics_name) if last_val_sticky: - # array = shm.array[array_key] - # if len(array): - # value = array[-1] last = last_row[array_key] last_val_sticky.update_from_data(-1, last) @@ -246,20 +242,18 @@ async def run_fsp_ui( chart.draw_curve( name=name, - data=shm.array, + shm=shm, overlay=True, color='default_light', array_key=name, **conf.get('chart_kwargs', {}) ) - # specially store ref to shm for lookup in display loop - chart._flows[name].shm = shm else: # create a new sub-chart widget for this fsp chart = linkedsplits.add_plot( name=name, - array=shm.array, + shm=shm, array_key=name, sidepane=sidepane, @@ -271,12 +265,6 @@ async def run_fsp_ui( **conf.get('chart_kwargs', {}) ) - # XXX: ONLY for sub-chart fsps, overlays have their - # data looked up from the chart's internal array set. - # TODO: we must get a data view api going STAT!! - chart._shm = shm - chart._flows[chart.data_key].shm = shm - # should **not** be the same sub-chart widget assert chart.name != linkedsplits.chart.name @@ -626,7 +614,7 @@ async def open_vlm_displays( shm = ohlcv chart = linked.add_plot( name='volume', - array=shm.array, + shm=shm, array_key='volume', sidepane=sidepane, @@ -639,7 +627,6 @@ async def open_vlm_displays( # the curve item internals are pretty convoluted. style='step', ) - chart._flows['volume'].shm = ohlcv # force 0 to always be in view def maxmin( @@ -666,11 +653,6 @@ def maxmin( # chart.hideAxis('right') # chart.showAxis('left') - # XXX: ONLY for sub-chart fsps, overlays have their - # data looked up from the chart's internal array set. - # TODO: we must get a data view api going STAT!! - chart._shm = shm - # send back new chart to caller task_status.started(chart) @@ -685,9 +667,9 @@ def maxmin( last_val_sticky.update_from_data(-1, value) - vlm_curve = chart.update_graphics_from_array( + vlm_curve = chart.update_graphics_from_flow( 'volume', - shm.array, + # shm.array, ) # size view to data once at outset @@ -795,9 +777,8 @@ def chart_curves( color = 'bracket' curve, _ = chart.draw_curve( - # name='dolla_vlm', name=name, - data=shm.array, + shm=shm, array_key=name, overlay=pi, color=color, @@ -812,7 +793,6 @@ def chart_curves( # ``.draw_curve()``. flow = chart._flows[name] assert flow.plot is pi - flow.shm = shm chart_curves( fields, @@ -847,7 +827,9 @@ def chart_curves( # liquidity events (well at least on low OHLC periods - 1s). vlm_curve.hide() chart.removeItem(vlm_curve) - chart._flows.pop('volume') + vflow = chart._flows['volume'] + vflow.render = False + # avoid range sorting on volume once disabled chart.view.disable_auto_yrange() From d0af280a59cf8a0af382609ecffa1820ad8eb6d6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 14 Apr 2022 10:10:38 -0400 Subject: [PATCH 003/113] Port view downsampling handler to new update apis --- piker/ui/_interaction.py | 89 +++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4872f5950..4b3bbb45d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -492,9 +492,9 @@ def wheelEvent( log.debug("Max zoom bruh...") return - if ev.delta() < 0 and vl >= len(chart._arrays[chart.name]) + 666: - log.debug("Min zoom bruh...") - return + # if ev.delta() < 0 and vl >= len(chart._flows[chart.name].shm.array) + 666: + # log.debug("Min zoom bruh...") + # return # actual scaling factor s = 1.015 ** (ev.delta() * -1 / 20) # self.state['wheelScaleFactor']) @@ -777,9 +777,15 @@ def _set_yrange( # calculate max, min y values in viewable x-range from data. # Make sure min bars/datums on screen is adhered. - else: - br = bars_range or chart.bars_range() - profiler(f'got bars range: {br}') + # else: + # TODO: eventually we should point to the + # ``FlowsTable`` (or wtv) which should perform + # the group operations? + + # flow = chart._flows[name or chart.name] + # br = bars_range or chart.bars_range() + # br = bars_range or chart.bars_range() + # profiler(f'got bars range: {br}') # TODO: maybe should be a method on the # chart widget/item? @@ -830,6 +836,8 @@ def _set_yrange( self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') + profiler.finish() + def enable_auto_yrange( self, src_vb: Optional[ChartView] = None, @@ -890,7 +898,7 @@ def x_uppx(self) -> float: graphics items which are our children. ''' - graphics = list(self._chart._graphics.values()) + graphics = [f.graphics for f in self._chart._flows.values()] if not graphics: return 0 @@ -903,44 +911,49 @@ def x_uppx(self) -> float: def maybe_downsample_graphics(self): + profiler = pg.debug.Profiler( + disabled=not pg_profile_enabled(), + gt=3, + ) + uppx = self.x_uppx() - if ( + if not ( # we probably want to drop this once we are "drawing in # view" for downsampled flows.. uppx and uppx > 16 and self._ic is not None ): + + # TODO: a faster single-loop-iterator way of doing this XD + chart = self._chart + linked = self.linkedsplits + plots = linked.subplots | {chart.name: chart} + for chart_name, chart in plots.items(): + for name, flow in chart._flows.items(): + + if not flow.render: + continue + + graphics = flow.graphics + + use_vr = False + if isinstance(graphics, BarItems): + use_vr = True + + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_flow( + name, + use_vr=use_vr, + + # gets passed down into graphics obj + profiler=profiler, + ) + + profiler(f'range change updated {chart_name}:{name}') + else: # don't bother updating since we're zoomed out bigly and # in a pan-interaction, in which case we shouldn't be # doing view-range based rendering (at least not yet). # print(f'{uppx} exiting early!') - return - - profiler = pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=3, - delayed=True, - ) - - # TODO: a faster single-loop-iterator way of doing this XD - chart = self._chart - linked = self.linkedsplits - plots = linked.subplots | {chart.name: chart} - for chart_name, chart in plots.items(): - for name, flow in chart._flows.items(): - graphics = flow.graphics - - use_vr = False - if isinstance(graphics, BarItems): - use_vr = True - - # pass in no array which will read and render from the last - # passed array (normally provided by the display loop.) - chart.update_graphics_from_array( - name, - use_vr=use_vr, - profiler=profiler, - ) - profiler(f'range change updated {chart_name}:{name}') - - profiler.finish() + profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') From 5a9bab0b690a688e6412d75b0d328596ae36f4ea Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 16 Apr 2022 15:22:11 -0400 Subject: [PATCH 004/113] WIP incremental render apis --- piker/ui/_curve.py | 8 +- piker/ui/_display.py | 1 - piker/ui/_flows.py | 224 ++++++++++++++++++++++++++++++++++++------- piker/ui/_ohlc.py | 16 ++-- 4 files changed, 198 insertions(+), 51 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 00a4ca7aa..871e55f58 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -290,9 +290,9 @@ def update_from_array( # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. - self.xData = x - self.yData = y - self._x, self._y = x, y + # self.xData = x + # self.yData = y + # self._x, self._y = x, y if view_range: profiler(f'view range slice {view_range}') @@ -328,7 +328,7 @@ def update_from_array( # x_last = x_iv[-1] # y_last = y_iv[-1] - self._last_vr = view_range + # self._last_vr = view_range # self.disable_cache() # flip_cache = True diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 4b695b04a..fda3fb042 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -437,7 +437,6 @@ def graphics_update_cycle( for curve_name, flow in vlm_chart._flows.items(): if not flow.render: - print(f'skipping flow {curve_name}?') continue update_fsp_chart( diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a9eb6a4f3..d5a0d1e19 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -22,6 +22,7 @@ incremental update. ''' +from __future__ import annotations from typing import ( Optional, Callable, @@ -36,8 +37,16 @@ ShmArray, # attach_shm_array ) -from ._ohlc import BarItems - +from ._ohlc import ( + BarItems, + gen_qpath, +) +from ._curve import ( + FastAppendCurve, +) +from ._compression import ( + ohlc_flatten, +) # class FlowsTable(msgspec.Struct): # ''' @@ -48,6 +57,20 @@ # ''' # flows: dict[str, np.ndarray] = {} +# @classmethod +# def from_token( +# cls, +# shm_token: tuple[ +# str, +# str, +# tuple[str, str], +# ], + +# ) -> Renderer: + +# shm = attach_shm_array(token) +# return cls(shm) + class Flow(msgspec.Struct): # , frozen=True): ''' @@ -61,16 +84,28 @@ class Flow(msgspec.Struct): # , frozen=True): ''' name: str plot: pg.PlotItem + graphics: pg.GraphicsObject + _shm: ShmArray + is_ohlc: bool = False render: bool = True # toggle for display loop - graphics: pg.GraphicsObject + _last_uppx: float = 0 + _in_ds: bool = False + + _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None + + # map from uppx -> (downsampled data, incremental graphics) + _render_table: dict[ + Optional[int], + tuple[Renderer, pg.GraphicsItem], + ] = {} # TODO: hackery to be able to set a shm later # but whilst also allowing this type to hashable, # likely will require serializable token that is used to attach # to the underlying shm ref after startup? - _shm: Optional[ShmArray] = None # currently, may be filled in "later" + # _shm: Optional[ShmArray] = None # currently, may be filled in "later" # last read from shm (usually due to an update call) _last_read: Optional[np.ndarray] = None @@ -219,7 +254,7 @@ def update_graphics( ''' # shm read and slice to view - xfirst, xlast, array, ivl, ivr, in_view = self.read() + read = xfirst, xlast, array, ivl, ivr, in_view = self.read() if ( not in_view.size @@ -227,10 +262,48 @@ def update_graphics( ): return self.graphics - array_key = array_key or self.name - graphics = self.graphics if isinstance(graphics, BarItems): + + # ugh, not luvin dis, should we have just a designated + # instance var? + r = self._render_table.get('src') + if not r: + r = Renderer( + flow=self, + draw=gen_qpath, # TODO: rename this to something with ohlc + last_read=read, + ) + self._render_table['src'] = (r, graphics) + + ds_curve_r = Renderer( + flow=self, + draw=gen_qpath, # TODO: rename this to something with ohlc + last_read=read, + prerender_fn=ohlc_flatten, + ) + + # baseline "line" downsampled OHLC curve that should + # kick on only when we reach a certain uppx threshold. + self._render_table[0] = ( + ds_curve_r, + FastAppendCurve( + y=y, + x=x, + name='OHLC', + color=self._color, + ), + ) + + # do checks for whether or not we require downsampling: + # - if we're **not** downsampling then we simply want to + # render the bars graphics curve and update.. + # - if insteam we are in a downsamplig state then we to + # update our pre-downsample-ready data and then pass that + # new data the downsampler algo for incremental update. + else: + # do incremental update + graphics.update_from_array( array, in_view, @@ -239,7 +312,55 @@ def update_graphics( **kwargs, ) + # generate and apply path to graphics obj + graphics.path, last = r.render(only_in_view=True) + graphics.draw_last(last) + else: + # should_ds = False + # should_redraw = False + + # # downsampling incremental state checking + # uppx = bars.x_uppx() + # px_width = bars.px_width() + # uppx_diff = (uppx - self._last_uppx) + + # if self.renderer is None: + # self.renderer = Renderer( + # flow=self, + + # if not self._in_ds: + # # in not currently marked as downsampling graphics + # # then only draw the full bars graphic for datums "in + # # view". + + # # check for downsampling conditions + # if ( + # # std m4 downsample conditions + # px_width + # and uppx_diff >= 4 + # or uppx_diff <= -3 + # or self._step_mode and abs(uppx_diff) >= 4 + + # ): + # log.info( + # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' + # ) + # self._last_uppx = uppx + # should_ds = True + + # elif ( + # uppx <= 2 + # and self._in_ds + # ): + # # we should de-downsample back to our original + # # source data so we clear our path data in prep + # # to generate a new one from original source data. + # should_redraw = True + # should_ds = False + + array_key = array_key or self.name + graphics.update_from_array( x=array['index'], y=array[array_key], @@ -253,51 +374,80 @@ def update_graphics( return graphics - # @classmethod - # def from_token( - # cls, - # shm_token: tuple[ - # str, - # str, - # tuple[str, str], - # ], - - # ) -> PathRenderer: - # shm = attach_shm_array(token) - # return cls(shm) +class Renderer(msgspec.Struct): + flow: Flow -class PathRenderer(msgspec.Struct): + # called to render path graphics + draw: Callable[np.ndarray, QPainterPath] - # output graphics rendering - path: Optional[QPainterPath] = None - - last_read_src_array: np.ndarray # called on input data but before - prerender_fn: Callable[ShmArray, np.ndarray] + prerender_fn: Optional[Callable[ShmArray, np.ndarray]] = None - def diff( - self, - ) -> dict[str, np.ndarray]: - ... + prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - def update(self) -> QPainterPath: - ''' - Incrementally update the internal path graphics from - updates in shm data and deliver the new (sub)-path - generated. + # last array view read + last_read: Optional[np.ndarray] = None - ''' - ... + # output graphics rendering + path: Optional[QPainterPath] = None + # def diff( + # self, + # latest_read: tuple[np.ndarray], + + # ) -> tuple[np.ndarray]: + # # blah blah blah + # # do diffing for prepend, append and last entry + # return ( + # to_prepend + # to_append + # last, + # ) def render( self, + # only render datums "in view" of the ``ChartView`` + only_in_view: bool = True, + ) -> list[QPainterPath]: ''' Render the current graphics path(s) ''' - ... + # do full source data render to path + xfirst, xlast, array, ivl, ivr, in_view = self.last_read + + if only_in_view: + # get latest data from flow shm + self.last_read = ( + xfirst, xlast, array, ivl, ivr, in_view + ) = self.flow.read() + + array = in_view + + if self.path is None or in_view: + # redraw the entire source data if we have either of: + # - no prior path graphic rendered or, + # - we always intend to re-render the data only in view + + if self.prerender_fn: + array = self.prerender_fn(array) + + hist, last = array[:-1], array[-1] + + # call path render func on history + self.path = self.draw(hist) + + elif self.path: + print(f'inremental update not supported yet {self.flow.name}') + # TODO: do incremental update + # prepend, append, last = self.diff(self.flow.read()) + + # do path generation for each segment + # and then push into graphics object. + + return self.path, last diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 44dbb0c24..bf56f5f56 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -46,7 +46,7 @@ def bar_from_ohlc_row( row: np.ndarray, - w: float + w: float = 0.43 ) -> tuple[QLineF]: ''' @@ -158,8 +158,9 @@ def path_arrays_from_ohlc( def gen_qpath( data: np.ndarray, - start: int, # XXX: do we need this? - w: float, + start: int = 0, # XXX: do we need this? + # 0.5 is no overlap between arms, 1.0 is full overlap + w: float = 0.43, path: Optional[QtGui.QPainterPath] = None, ) -> QtGui.QPainterPath: @@ -310,7 +311,7 @@ def draw_from_data( self._pi.addItem(curve) self._ds_line = curve - self._ds_xrange = (index[0], index[-1]) + # self._ds_xrange = (index[0], index[-1]) # trigger render # https://doc.qt.io/qt-5/qgraphicsitem.html#update @@ -358,7 +359,7 @@ def update_from_array( # index = self.start_index istart, istop = self._xrange - ds_istart, ds_istop = self._ds_xrange + # ds_istart, ds_istop = self._ds_xrange index = ohlc['index'] first_index, last_index = index[0], index[-1] @@ -435,9 +436,6 @@ def update_from_array( # stop here since we don't need to update bars path any more # as we delegate to the downsample line with updates. - profiler.finish() - # print('terminating early') - return else: # we should be in bars mode @@ -606,7 +604,7 @@ def update_from_array( if flip_cache: self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - profiler.finish() + # profiler.finish() def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From e0a72a217405f23ee3a37ca6dd9bf8d003cb24e6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 18 Apr 2022 08:30:28 -0400 Subject: [PATCH 005/113] WIP starting architecture doc str writeup.. --- piker/ui/_flows.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d5a0d1e19..772aa0267 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -302,6 +302,7 @@ def update_graphics( # update our pre-downsample-ready data and then pass that # new data the downsampler algo for incremental update. else: + pass # do incremental update graphics.update_from_array( @@ -417,6 +418,14 @@ def render( ''' Render the current graphics path(s) + There are (at least) 3 stages from source data to graphics data: + - a data transform (which can be stored in additional shm) + - a graphics transform which converts discrete basis data to + a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``, + ``step_path_arrays_from_1d()``, etc.) + + - blah blah blah (from notes) + ''' # do full source data render to path xfirst, xlast, array, ivl, ivr, in_view = self.last_read From f4dc0fbab8918bb63c7b302379036d82da532606 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:42:49 -0400 Subject: [PATCH 006/113] Add `BarItems.draw_last()` and disable `.update_from_array()` --- piker/ui/_ohlc.py | 602 ++++++++++++++++++++++++---------------------- 1 file changed, 321 insertions(+), 281 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index bf56f5f56..328d62b9e 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -244,7 +244,7 @@ def __init__( self.fast_path = QtGui.QPainterPath() self._xrange: tuple[int, int] - self._yrange: tuple[float, float] + # self._yrange: tuple[float, float] self._vrange = None # TODO: don't render the full backing array each time @@ -281,10 +281,10 @@ def draw_from_data( # self.start_index = len(ohlc) index = ohlc['index'] self._xrange = (index[0], index[-1]) - self._yrange = ( - np.nanmax(ohlc['high']), - np.nanmin(ohlc['low']), - ) + # self._yrange = ( + # np.nanmax(ohlc['high']), + # np.nanmin(ohlc['low']), + # ) # up to last to avoid double draw of last bar self._last_bar_lines = bar_from_ohlc_row(last, self.w) @@ -325,286 +325,326 @@ def x_uppx(self) -> int: else: return 0 - def update_from_array( + # def update_from_array( + # self, + + # # full array input history + # ohlc: np.ndarray, + + # # pre-sliced array data that's "in view" + # ohlc_iv: np.ndarray, + + # view_range: Optional[tuple[int, int]] = None, + # profiler: Optional[pg.debug.Profiler] = None, + + # ) -> None: + # ''' + # Update the last datum's bar graphic from input data array. + + # This routine should be interface compatible with + # ``pg.PlotCurveItem.setData()``. Normally this method in + # ``pyqtgraph`` seems to update all the data passed to the + # graphics object, and then update/rerender, but here we're + # assuming the prior graphics havent changed (OHLC history rarely + # does) so this "should" be simpler and faster. + + # This routine should be made (transitively) as fast as possible. + + # ''' + # profiler = profiler or pg.debug.Profiler( + # disabled=not pg_profile_enabled(), + # gt=ms_slower_then, + # delayed=True, + # ) + + # # index = self.start_index + # istart, istop = self._xrange + # # ds_istart, ds_istop = self._ds_xrange + + # index = ohlc['index'] + # first_index, last_index = index[0], index[-1] + + # # length = len(ohlc) + # # prepend_length = istart - first_index + # # append_length = last_index - istop + + # # ds_prepend_length = ds_istart - first_index + # # ds_append_length = last_index - ds_istop + + # flip_cache = False + + # x_gt = 16 + # if self._ds_line: + # uppx = self._ds_line.x_uppx() + # else: + # uppx = 0 + + # should_line = self._in_ds + # if ( + # self._in_ds + # and uppx < x_gt + # ): + # should_line = False + + # elif ( + # not self._in_ds + # and uppx >= x_gt + # ): + # should_line = True + + # profiler('ds logic complete') + + # if should_line: + # # update the line graphic + # # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) + # x, y = self._ds_line_xy = ohlc_flatten(ohlc) + # x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) + # profiler('flattening bars to line') + + # # TODO: we should be diffing the amount of new data which + # # needs to be downsampled. Ideally we actually are just + # # doing all the ds-ing in sibling actors so that the data + # # can just be read and rendered to graphics on events of our + # # choice. + # # diff = do_diff(ohlc, new_bit) + # curve = self._ds_line + # curve.update_from_array( + # x=x, + # y=y, + # x_iv=x_iv, + # y_iv=y_iv, + # view_range=None, # hack + # profiler=profiler, + # ) + # profiler('updated ds line') + + # if not self._in_ds: + # # hide bars and show line + # self.hide() + # # XXX: is this actually any faster? + # # self._pi.removeItem(self) + + # # TODO: a `.ui()` log level? + # log.info( + # f'downsampling to line graphic {self._name}' + # ) + + # # self._pi.addItem(curve) + # curve.show() + # curve.update() + # self._in_ds = True + + # # stop here since we don't need to update bars path any more + # # as we delegate to the downsample line with updates. + + # else: + # # we should be in bars mode + + # if self._in_ds: + # # flip back to bars graphics and hide the downsample line. + # log.info(f'showing bars graphic {self._name}') + + # curve = self._ds_line + # curve.hide() + # # self._pi.removeItem(curve) + + # # XXX: is this actually any faster? + # # self._pi.addItem(self) + # self.show() + # self._in_ds = False + + # # generate in_view path + # self.path = gen_qpath( + # ohlc_iv, + # 0, + # self.w, + # # path=self.path, + # ) + + # # TODO: to make the downsampling faster + # # - allow mapping only a range of lines thus only drawing as + # # many bars as exactly specified. + # # - move ohlc "flattening" to a shmarr + # # - maybe move all this embedded logic to a higher + # # level type? + + # # if prepend_length: + # # # new history was added and we need to render a new path + # # prepend_bars = ohlc[:prepend_length] + + # # if ds_prepend_length: + # # ds_prepend_bars = ohlc[:ds_prepend_length] + # # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) + # # fx = np.concatenate((pre_x, fx)) + # # fy = np.concatenate((pre_y, fy)) + # # profiler('ds line prepend diff complete') + + # # if append_length: + # # # generate new graphics to match provided array + # # # path appending logic: + # # # we need to get the previous "current bar(s)" for the time step + # # # and convert it to a sub-path to append to the historical set + # # # new_bars = ohlc[istop - 1:istop + append_length - 1] + # # append_bars = ohlc[-append_length - 1:-1] + # # # print(f'ohlc bars to append size: {append_bars.size}\n') + + # # if ds_append_length: + # # ds_append_bars = ohlc[-ds_append_length - 1:-1] + # # post_x, post_y = ohlc_flatten(ds_append_bars) + # # print( + # # f'ds curve to append sizes: {(post_x.size, post_y.size)}' + # # ) + # # fx = np.concatenate((fx, post_x)) + # # fy = np.concatenate((fy, post_y)) + + # # profiler('ds line append diff complete') + + # profiler('array diffs complete') + + # # does this work? + # last = ohlc[-1] + # # fy[-1] = last['close'] + + # # # incremental update and cache line datums + # # self._ds_line_xy = fx, fy + + # # maybe downsample to line + # # ds = self.maybe_downsample() + # # if ds: + # # # if we downsample to a line don't bother with + # # # any more path generation / updates + # # self._ds_xrange = first_index, last_index + # # profiler('downsampled to line') + # # return + + # # print(in_view.size) + + # # if self.path: + # # self.path = path + # # self.path.reserve(path.capacity()) + # # self.path.swap(path) + + # # path updates + # # if prepend_length: + # # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path + # # # y value not matching the first value from + # # # ohlc[prepend_length + 1] ??? + # # prepend_path = gen_qpath(prepend_bars, 0, self.w) + # # old_path = self.path + # # self.path = prepend_path + # # self.path.addPath(old_path) + # # profiler('path PREPEND') + + # # if append_length: + # # append_path = gen_qpath(append_bars, 0, self.w) + + # # self.path.moveTo( + # # float(istop - self.w), + # # float(append_bars[0]['open']) + # # ) + # # self.path.addPath(append_path) + + # # profiler('path APPEND') + # # fp = self.fast_path + # # if fp is None: + # # self.fast_path = append_path + + # # else: + # # fp.moveTo( + # # float(istop - self.w), float(new_bars[0]['open']) + # # ) + # # fp.addPath(append_path) + + # # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # # flip_cache = True + + # self._xrange = first_index, last_index + + # # trigger redraw despite caching + # self.prepareGeometryChange() + + # self.draw_last(last) + + # # # generate new lines objects for updatable "current bar" + # # self._last_bar_lines = bar_from_ohlc_row(last, self.w) + + # # # last bar update + # # i, o, h, l, last, v = last[ + # # ['index', 'open', 'high', 'low', 'close', 'volume'] + # # ] + # # # assert i == self.start_index - 1 + # # # assert i == last_index + # # body, larm, rarm = self._last_bar_lines + + # # # XXX: is there a faster way to modify this? + # # rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # # # writer is responsible for changing open on "first" volume of bar + # # larm.setLine(larm.x1(), o, larm.x2(), o) + + # # if l != h: # noqa + + # # if body is None: + # # body = self._last_bar_lines[0] = QLineF(i, l, i, h) + # # else: + # # # update body + # # body.setLine(i, l, i, h) + + # # # XXX: pretty sure this is causing an issue where the bar has + # # # a large upward move right before the next sample and the body + # # # is getting set to None since the next bar is flat but the shm + # # # array index update wasn't read by the time this code runs. Iow + # # # we're doing this removal of the body for a bar index that is + # # # now out of date / from some previous sample. It's weird + # # # though because i've seen it do this to bars i - 3 back? + + # profiler('last bar set') + + # self.update() + # profiler('.update()') + + # if flip_cache: + # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # # profiler.finish() + + def draw_last( self, - - # full array input history - ohlc: np.ndarray, - - # pre-sliced array data that's "in view" - ohlc_iv: np.ndarray, - - view_range: Optional[tuple[int, int]] = None, - profiler: Optional[pg.debug.Profiler] = None, + last: np.ndarray, ) -> None: - ''' - Update the last datum's bar graphic from input data array. - - This routine should be interface compatible with - ``pg.PlotCurveItem.setData()``. Normally this method in - ``pyqtgraph`` seems to update all the data passed to the - graphics object, and then update/rerender, but here we're - assuming the prior graphics havent changed (OHLC history rarely - does) so this "should" be simpler and faster. - - This routine should be made (transitively) as fast as possible. - - ''' - profiler = profiler or pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=ms_slower_then, - delayed=True, - ) - - # index = self.start_index - istart, istop = self._xrange - # ds_istart, ds_istop = self._ds_xrange - - index = ohlc['index'] - first_index, last_index = index[0], index[-1] - - # length = len(ohlc) - # prepend_length = istart - first_index - # append_length = last_index - istop - - # ds_prepend_length = ds_istart - first_index - # ds_append_length = last_index - ds_istop - - flip_cache = False - - x_gt = 16 - if self._ds_line: - uppx = self._ds_line.x_uppx() - else: - uppx = 0 - - should_line = self._in_ds - if ( - self._in_ds - and uppx < x_gt - ): - should_line = False - - elif ( - not self._in_ds - and uppx >= x_gt - ): - should_line = True - - profiler('ds logic complete') - - if should_line: - # update the line graphic - # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) - x, y = self._ds_line_xy = ohlc_flatten(ohlc) - x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) - profiler('flattening bars to line') - - # TODO: we should be diffing the amount of new data which - # needs to be downsampled. Ideally we actually are just - # doing all the ds-ing in sibling actors so that the data - # can just be read and rendered to graphics on events of our - # choice. - # diff = do_diff(ohlc, new_bit) - curve = self._ds_line - curve.update_from_array( - x=x, - y=y, - x_iv=x_iv, - y_iv=y_iv, - view_range=None, # hack - profiler=profiler, - ) - profiler('updated ds line') - - if not self._in_ds: - # hide bars and show line - self.hide() - # XXX: is this actually any faster? - # self._pi.removeItem(self) - - # TODO: a `.ui()` log level? - log.info( - f'downsampling to line graphic {self._name}' - ) - - # self._pi.addItem(curve) - curve.show() - curve.update() - self._in_ds = True - - # stop here since we don't need to update bars path any more - # as we delegate to the downsample line with updates. - - else: - # we should be in bars mode - - if self._in_ds: - # flip back to bars graphics and hide the downsample line. - log.info(f'showing bars graphic {self._name}') - - curve = self._ds_line - curve.hide() - # self._pi.removeItem(curve) - - # XXX: is this actually any faster? - # self._pi.addItem(self) - self.show() - self._in_ds = False - - # generate in_view path - self.path = gen_qpath( - ohlc_iv, - 0, - self.w, - # path=self.path, - ) + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last, self.w) - # TODO: to make the downsampling faster - # - allow mapping only a range of lines thus only drawing as - # many bars as exactly specified. - # - move ohlc "flattening" to a shmarr - # - maybe move all this embedded logic to a higher - # level type? - - # if prepend_length: - # # new history was added and we need to render a new path - # prepend_bars = ohlc[:prepend_length] - - # if ds_prepend_length: - # ds_prepend_bars = ohlc[:ds_prepend_length] - # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - # fx = np.concatenate((pre_x, fx)) - # fy = np.concatenate((pre_y, fy)) - # profiler('ds line prepend diff complete') - - # if append_length: - # # generate new graphics to match provided array - # # path appending logic: - # # we need to get the previous "current bar(s)" for the time step - # # and convert it to a sub-path to append to the historical set - # # new_bars = ohlc[istop - 1:istop + append_length - 1] - # append_bars = ohlc[-append_length - 1:-1] - # # print(f'ohlc bars to append size: {append_bars.size}\n') - - # if ds_append_length: - # ds_append_bars = ohlc[-ds_append_length - 1:-1] - # post_x, post_y = ohlc_flatten(ds_append_bars) - # print( - # f'ds curve to append sizes: {(post_x.size, post_y.size)}' - # ) - # fx = np.concatenate((fx, post_x)) - # fy = np.concatenate((fy, post_y)) - - # profiler('ds line append diff complete') - - profiler('array diffs complete') - - # does this work? - last = ohlc[-1] - # fy[-1] = last['close'] - - # # incremental update and cache line datums - # self._ds_line_xy = fx, fy - - # maybe downsample to line - # ds = self.maybe_downsample() - # if ds: - # # if we downsample to a line don't bother with - # # any more path generation / updates - # self._ds_xrange = first_index, last_index - # profiler('downsampled to line') - # return - - # print(in_view.size) - - # if self.path: - # self.path = path - # self.path.reserve(path.capacity()) - # self.path.swap(path) - - # path updates - # if prepend_length: - # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # # y value not matching the first value from - # # ohlc[prepend_length + 1] ??? - # prepend_path = gen_qpath(prepend_bars, 0, self.w) - # old_path = self.path - # self.path = prepend_path - # self.path.addPath(old_path) - # profiler('path PREPEND') - - # if append_length: - # append_path = gen_qpath(append_bars, 0, self.w) - - # self.path.moveTo( - # float(istop - self.w), - # float(append_bars[0]['open']) - # ) - # self.path.addPath(append_path) - - # profiler('path APPEND') - # fp = self.fast_path - # if fp is None: - # self.fast_path = append_path - - # else: - # fp.moveTo( - # float(istop - self.w), float(new_bars[0]['open']) - # ) - # fp.addPath(append_path) - - # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # flip_cache = True - - self._xrange = first_index, last_index - - # trigger redraw despite caching - self.prepareGeometryChange() - - # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == self.start_index - 1 - # assert i == last_index - body, larm, rarm = self._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = self._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the bar has - # a large upward move right before the next sample and the body - # is getting set to None since the next bar is flat but the shm - # array index update wasn't read by the time this code runs. Iow - # we're doing this removal of the body for a bar index that is - # now out of date / from some previous sample. It's weird - # though because i've seen it do this to bars i - 3 back? - - profiler('last bar set') - - self.update() - profiler('.update()') - - if flip_cache: - self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # profiler.finish() + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == self.start_index - 1 + # assert i == last_index + body, larm, rarm = self._last_bar_lines + + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) + + if l != h: # noqa + + if body is None: + body = self._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) + + # XXX: pretty sure this is causing an issue where the bar has + # a large upward move right before the next sample and the body + # is getting set to None since the next bar is flat but the shm + # array index update wasn't read by the time this code runs. Iow + # we're doing this removal of the body for a bar index that is + # now out of date / from some previous sample. It's weird + # though because i've seen it do this to bars i - 3 back? def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From 427a33654b4e308fb7fafc6ef8e0cb489809b04f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:43:47 -0400 Subject: [PATCH 007/113] More WIP, implement `BarItems` rendering in `Flow.update_graphics()` --- piker/ui/_flows.py | 404 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 314 insertions(+), 90 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 772aa0267..8d5e7e773 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,6 +23,8 @@ ''' from __future__ import annotations +from functools import partial +import time from typing import ( Optional, Callable, @@ -30,6 +32,7 @@ import msgspec import numpy as np +from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath @@ -37,6 +40,7 @@ ShmArray, # attach_shm_array ) +from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( BarItems, gen_qpath, @@ -46,7 +50,12 @@ ) from ._compression import ( ohlc_flatten, + ds_m4, ) +from ..log import get_logger + + +log = get_logger(__name__) # class FlowsTable(msgspec.Struct): # ''' @@ -72,11 +81,63 @@ # return cls(shm) +def rowarr_to_path( + rows_array: np.ndarray, + x_basis: np.ndarray, + flow: Flow, + +) -> QPainterPath: + + # TODO: we could in theory use ``numba`` to flatten + # if needed? + + # to 1d + y = rows_array.flatten() + + return pg.functions.arrayToQPath( + # these get passed at render call time + x=x_basis[:y.size], + y=y, + connect='all', + finiteCheck=False, + path=flow.path, + ) + + +def ohlc_flat_view( + ohlc_shm: ShmArray, + + # XXX: we bind this in currently.. + x_basis: np.ndarray, + + # vr: Optional[slice] = None, + +) -> np.ndarray: + ''' + Return flattened-non-copy view into an OHLC shm array. + + ''' + ohlc = ohlc_shm._array[['open', 'high', 'low', 'close']] + # if vr: + # ohlc = ohlc[vr] + # x = x_basis[vr] + + unstructured = rfn.structured_to_unstructured( + ohlc, + copy=False, + ) + # breakpoint() + y = unstructured.flatten() + x = x_basis[:y.size] + return x, y + + class Flow(msgspec.Struct): # , frozen=True): ''' - (FinancialSignal-)Flow compound type which wraps a real-time - graphics (curve) and its backing data stream together for high level - access and control. + (Financial Signal-)Flow compound type which wraps a real-time + shm array stream with displayed graphics (curves, charts) + for high level access and control as well as efficient incremental + update. The intention is for this type to eventually be capable of shm-passing of incrementally updated graphics stream data between actors. @@ -89,6 +150,8 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop + flat: Optional[ShmArray] = None + x_basis: Optional[np.ndarray] = None _last_uppx: float = 0 _in_ds: bool = False @@ -96,6 +159,7 @@ class Flow(msgspec.Struct): # , frozen=True): _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None # map from uppx -> (downsampled data, incremental graphics) + _src_r: Optional[Renderer] = None _render_table: dict[ Optional[int], tuple[Renderer, pg.GraphicsItem], @@ -215,7 +279,9 @@ def read(self) -> tuple[ int, int, np.ndarray, int, int, np.ndarray, ]: + # read call array = self.shm.array + indexes = array['index'] ifirst = indexes[0] ilast = indexes[-1] @@ -245,6 +311,8 @@ def update_graphics( render: bool = True, array_key: Optional[str] = None, + profiler=None, + **kwargs, ) -> pg.GraphicsObject: @@ -253,8 +321,19 @@ def update_graphics( render to graphics. ''' + + profiler = profiler or pg.debug.Profiler( + msg=f'Flow.update_graphics() for {self.name}', + disabled=not pg_profile_enabled(), + gt=ms_slower_then, + delayed=True, + ) # shm read and slice to view - read = xfirst, xlast, array, ivl, ivr, in_view = self.read() + read = ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = self.read() + profiler('read src shm data') if ( not in_view.size @@ -265,100 +344,182 @@ def update_graphics( graphics = self.graphics if isinstance(graphics, BarItems): - # ugh, not luvin dis, should we have just a designated - # instance var? - r = self._render_table.get('src') + # if no source data renderer exists create one. + r = self._src_r if not r: - r = Renderer( + # OHLC bars path renderer + r = self._src_r = Renderer( flow=self, - draw=gen_qpath, # TODO: rename this to something with ohlc + # TODO: rename this to something with ohlc + draw_path=gen_qpath, last_read=read, ) - self._render_table['src'] = (r, graphics) + + # create a flattened view onto the OHLC array + # which can be read as a line-style format + # shm = self.shm + # self.flat = shm.unstruct_view(['open', 'high', 'low', 'close']) + # import pdbpp + # pdbpp.set_trace() + # x = self.x_basis = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # # self.flat._array.shape, + # self.flat.shape, + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) ds_curve_r = Renderer( flow=self, - draw=gen_qpath, # TODO: rename this to something with ohlc + + # just swap in the flat view + data_t=lambda array: self.flat.array, + # data_t=partial( + # ohlc_flat_view, + # self.shm, + # ), last_read=read, - prerender_fn=ohlc_flatten, + draw_path=partial( + rowarr_to_path, + x_basis=None, + ), + + ) + curve = FastAppendCurve( + # y=y, + # x=x, + name='OHLC', + color=graphics._color, ) + curve.hide() + self.plot.addItem(curve) # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. self._render_table[0] = ( ds_curve_r, - FastAppendCurve( - y=y, - x=x, - name='OHLC', - color=self._color, - ), + curve, ) + dsc_r, curve = self._render_table[0] + # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. # - if insteam we are in a downsamplig state then we to + x_gt = 8 + uppx = curve.x_uppx() + in_line = should_line = curve.isVisible() + if ( + should_line + and uppx < x_gt + ): + should_line = False + + elif ( + not should_line + and uppx >= x_gt + ): + should_line = True + + profiler(f'ds logic complete line={should_line}') + + # do graphics updates + if should_line: + # start = time.time() + # y = self.shm.unstruct_view( + # ['open', 'high', 'low', 'close'], + # ) + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # # start = self.flat._first.value + + # x = self.x_basis[:y.size].flatten() + # y = y.flatten() + # profiler('flattening bars to line') + # path, last = dsc_r.render(read) + # x, flat = ohlc_flat_view( + # ohlc_shm=self.shm, + # x_basis=x_basis, + # ) + # y = y.flatten() + # y_iv = y[ivl:ivr].flatten() + # x_iv = x[ivl:ivr].flatten() + # assert y.size == x.size + + x, y = self.flat = ohlc_flatten(array) + x_iv, y_iv = ohlc_flatten(in_view) + profiler('flattened OHLC data') + + curve.update_from_array( + x, + y, + x_iv=x_iv, + y_iv=y_iv, + view_range=None, # hack + profiler=profiler, + ) + profiler('updated ds curve') + + else: + # render incremental or in-view update + # and apply ouput (path) to graphics. + path, last = r.render( + read, + only_in_view=True, + ) + + graphics.path = path + graphics.draw_last(last) + + # NOTE: on appends we used to have to flip the coords + # cache thought it doesn't seem to be required any more? + # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + graphics.prepareGeometryChange() + graphics.update() + + if ( + not in_line + and should_line + ): + # change to line graphic + + log.info( + f'downsampling to line graphic {self.name}' + ) + graphics.hide() + # graphics.update() + curve.show() + curve.update() + + elif in_line and not should_line: + log.info(f'showing bars graphic {self.name}') + curve.hide() + graphics.show() + graphics.update() + # update our pre-downsample-ready data and then pass that # new data the downsampler algo for incremental update. - else: - pass - # do incremental update - graphics.update_from_array( - array, - in_view, - view_range=(ivl, ivr) if use_vr else None, + # graphics.update_from_array( + # array, + # in_view, + # view_range=(ivl, ivr) if use_vr else None, - **kwargs, - ) + # **kwargs, + # ) - # generate and apply path to graphics obj - graphics.path, last = r.render(only_in_view=True) - graphics.draw_last(last) + # generate and apply path to graphics obj + # graphics.path, last = r.render( + # read, + # only_in_view=True, + # ) + # graphics.draw_last(last) else: - # should_ds = False - # should_redraw = False - - # # downsampling incremental state checking - # uppx = bars.x_uppx() - # px_width = bars.px_width() - # uppx_diff = (uppx - self._last_uppx) - - # if self.renderer is None: - # self.renderer = Renderer( - # flow=self, - - # if not self._in_ds: - # # in not currently marked as downsampling graphics - # # then only draw the full bars graphic for datums "in - # # view". - - # # check for downsampling conditions - # if ( - # # std m4 downsample conditions - # px_width - # and uppx_diff >= 4 - # or uppx_diff <= -3 - # or self._step_mode and abs(uppx_diff) >= 4 - - # ): - # log.info( - # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - # ) - # self._last_uppx = uppx - # should_ds = True - - # elif ( - # uppx <= 2 - # and self._in_ds - # ): - # # we should de-downsample back to our original - # # source data so we clear our path data in prep - # # to generate a new one from original source data. - # should_redraw = True - # should_ds = False + # ``FastAppendCurve`` case: array_key = array_key or self.name @@ -376,23 +537,64 @@ def update_graphics( return graphics +def xy_downsample( + x, + y, + px_width, + uppx, + + x_spacer: float = 0.5, + +) -> tuple[np.ndarray, np.ndarray]: + + # downsample whenever more then 1 pixels per datum can be shown. + # always refresh data bounds until we get diffing + # working properly, see above.. + bins, x, y = ds_m4( + x, + y, + px_width=px_width, + uppx=uppx, + log_scale=bool(uppx) + ) + + # flatten output to 1d arrays suitable for path-graphics generation. + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array( + [-x_spacer, 0, 0, x_spacer] + )).flatten() + y = y.flatten() + + return x, y + + class Renderer(msgspec.Struct): flow: Flow # called to render path graphics - draw: Callable[np.ndarray, QPainterPath] + draw_path: Callable[np.ndarray, QPainterPath] - # called on input data but before - prerender_fn: Optional[Callable[ShmArray, np.ndarray]] = None + # called on input data but before any graphics format + # conversions or processing. + data_t: Optional[Callable[ShmArray, np.ndarray]] = None + data_t_shm: Optional[ShmArray] = None + # called on the final data (transform) output to convert + # to "graphical data form" a format that can be passed to + # the ``.draw()`` implementation. + graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None + graphics_t_shm: Optional[ShmArray] = None + + # path graphics update implementation methods prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None # last array view read last_read: Optional[np.ndarray] = None - # output graphics rendering + # output graphics rendering, the main object + # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None # def diff( @@ -411,8 +613,10 @@ class Renderer(msgspec.Struct): def render( self, + new_read, + # only render datums "in view" of the ``ChartView`` - only_in_view: bool = True, + only_in_view: bool = False, ) -> list[QPainterPath]: ''' @@ -428,28 +632,42 @@ def render( ''' # do full source data render to path - xfirst, xlast, array, ivl, ivr, in_view = self.last_read + last_read = ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = self.last_read if only_in_view: - # get latest data from flow shm - self.last_read = ( - xfirst, xlast, array, ivl, ivr, in_view - ) = self.flow.read() - array = in_view + # # get latest data from flow shm + # self.last_read = ( + # xfirst, xlast, array, ivl, ivr, in_view + # ) = new_read - if self.path is None or in_view: + if self.path is None or only_in_view: # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view - if self.prerender_fn: - array = self.prerender_fn(array) - - hist, last = array[:-1], array[-1] - - # call path render func on history - self.path = self.draw(hist) + # data transform: convert source data to a format + # expected to be incrementally updates and later rendered + # to a more graphics native format. + if self.data_t: + array = self.data_t(array) + + # maybe allocate shm for data transform output + # if self.data_t_shm is None: + # fshm = self.flow.shm + + # shm, opened = maybe_open_shm_array( + # f'{self.flow.name}_data_t', + # # TODO: create entry for each time frame + # dtype=array.dtype, + # readonly=False, + # ) + # assert opened + # shm.push(array) + # self.data_t_shm = shm elif self.path: print(f'inremental update not supported yet {self.flow.name}') @@ -459,4 +677,10 @@ def render( # do path generation for each segment # and then push into graphics object. + hist, last = array[:-1], array[-1] + + # call path render func on history + self.path = self.draw_path(hist) + + self.last_read = new_read return self.path, last From 239c9d701a5e6d3441ea02624001b3579b1eaac2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 11:44:27 -0400 Subject: [PATCH 008/113] Don't require data input to constructor --- piker/ui/_curve.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 871e55f58..c1c525de3 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -44,6 +44,7 @@ log = get_logger(__name__) +# TODO: numba this instead.. def step_path_arrays_from_1d( x: np.ndarray, y: np.ndarray, @@ -119,8 +120,8 @@ class FastAppendCurve(pg.GraphicsObject): def __init__( self, - x: np.ndarray, - y: np.ndarray, + x: np.ndarray = None, + y: np.ndarray = None, *args, step_mode: bool = False, @@ -461,6 +462,7 @@ def update_from_array( ): new_x = x[-append_length - 2:-1] new_y = y[-append_length - 2:-1] + profiler('sliced append path') if self._step_mode: new_x, new_y = step_path_arrays_from_1d( @@ -474,7 +476,12 @@ def update_from_array( new_x = new_x[1:] new_y = new_y[1:] - profiler('diffed append arrays') + profiler('generated step data') + + else: + profiler( + f'diffed array input, append_length={append_length}' + ) if should_ds: new_x, new_y = self.downsample( @@ -491,6 +498,7 @@ def update_from_array( finiteCheck=False, path=self.fast_path, ) + profiler(f'generated append qpath') if self.use_fpath: # an attempt at trying to make append-updates faster.. From 3dbce6f891c81cc868f7f77de0e350fa966696a5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 20 Apr 2022 12:13:18 -0400 Subject: [PATCH 009/113] Add `FastAppendCurve.draw_last()` --- piker/ui/_curve.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index c1c525de3..47132d7ce 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -337,9 +337,6 @@ def update_from_array( else: self._xrange = x[0], x[-1] - x_last = x[-1] - y_last = y[-1] - # check for downsampling conditions if ( # std m4 downsample conditions @@ -498,7 +495,7 @@ def update_from_array( finiteCheck=False, path=self.fast_path, ) - profiler(f'generated append qpath') + profiler('generated append qpath') if self.use_fpath: # an attempt at trying to make append-updates faster.. @@ -537,6 +534,28 @@ def update_from_array( # self.disable_cache() # flip_cache = True + self.draw_last(x, y) + profiler('draw last segment') + + # trigger redraw of path + # do update before reverting to cache mode + # self.prepareGeometryChange() + self.update() + profiler('.update()') + + # if flip_cache: + # # XXX: seems to be needed to avoid artifacts (see above). + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + + def draw_last( + self, + x: np.ndarray, + y: np.ndarray, + + ) -> None: + x_last = x[-1] + y_last = y[-1] + # draw the "current" step graphic segment so it lines up with # the "middle" of the current (OHLC) sample. if self._step_mode: @@ -556,21 +575,9 @@ def update_from_array( else: self._last_line = QLineF( x[-2], y[-2], - x[-1], y_last + x_last, y_last ) - profiler('draw last segment') - - # trigger redraw of path - # do update before reverting to cache mode - # self.prepareGeometryChange() - self.update() - profiler('.update()') - - # if flip_cache: - # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): From 7e1ec7b5a7de679396e2747f72f271aca5c5d3c8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 May 2022 17:57:14 -0400 Subject: [PATCH 010/113] Incrementally update flattend OHLC data After much effort (and exhaustion) but failure to get a view into our `numpy` OHLC struct-array, this instead allocates an in-thread-memory array which is updated with flattened data every flow update cycle. I need to report what I think is a bug to `numpy` core about the whole view thing not working but, more or less this gets the same behaviour and minimizes work to flatten the sampled data for line-graphics drawing thus improving refresh latency when drawing large downsampled curves. Update the OHLC ds curve with view aware data sliced out from the pre-allocated and incrementally updated data (we had to add a last index var `._iflat` to track appends - this should be moved into a renderer eventually?). --- piker/ui/_flows.py | 103 ++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 8d5e7e773..20ffc510f 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -24,7 +24,6 @@ ''' from __future__ import annotations from functools import partial -import time from typing import ( Optional, Callable, @@ -38,7 +37,7 @@ from ..data._sharedmem import ( ShmArray, - # attach_shm_array + open_shm_array, ) from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( @@ -152,6 +151,7 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop flat: Optional[ShmArray] = None x_basis: Optional[np.ndarray] = None + _iflat: int = 0 _last_uppx: float = 0 _in_ds: bool = False @@ -344,6 +344,7 @@ def update_graphics( graphics = self.graphics if isinstance(graphics, BarItems): + fields = ['open', 'high', 'low', 'close'] # if no source data renderer exists create one. r = self._src_r if not r: @@ -357,17 +358,37 @@ def update_graphics( # create a flattened view onto the OHLC array # which can be read as a line-style format - # shm = self.shm - # self.flat = shm.unstruct_view(['open', 'high', 'low', 'close']) + shm = self.shm + + # flat = self.flat = self.shm.unstruct_view(fields) + self.flat = self.shm.ustruct(fields) + self._iflat = self.shm._last.value + # import pdbpp # pdbpp.set_trace() - # x = self.x_basis = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # # self.flat._array.shape, - # self.flat.shape, - # ) + np.array([-0.5, 0, 0, 0.5]) + # assert len(flat._array) == len(self.shm._array[fields]) + + x = self.x_basis = ( + np.broadcast_to( + shm._array['index'][:, None], + ( + shm._array.size, + # 4, # only ohlc + self.flat.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + + # fshm = self.flat = open_shm_array( + # f'{self.name}_flat', + # dtype=flattened.dtype, + # size=flattened.size, # ) + # fshm.push(flattened) + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.flat._first.value ds_curve_r = Renderer( flow=self, @@ -407,7 +428,7 @@ def update_graphics( # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. # - if insteam we are in a downsamplig state then we to - x_gt = 8 + x_gt = 6 uppx = curve.x_uppx() in_line = should_line = curve.isVisible() if ( @@ -426,37 +447,43 @@ def update_graphics( # do graphics updates if should_line: - # start = time.time() - # y = self.shm.unstruct_view( - # ['open', 'high', 'low', 'close'], - # ) - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # # start = self.flat._first.value - - # x = self.x_basis[:y.size].flatten() - # y = y.flatten() - # profiler('flattening bars to line') - # path, last = dsc_r.render(read) - # x, flat = ohlc_flat_view( - # ohlc_shm=self.shm, - # x_basis=x_basis, - # ) - # y = y.flatten() - # y_iv = y[ivl:ivr].flatten() - # x_iv = x[ivl:ivr].flatten() - # assert y.size == x.size - x, y = self.flat = ohlc_flatten(array) - x_iv, y_iv = ohlc_flatten(in_view) - profiler('flattened OHLC data') + # update flatted ohlc copy + iflat, ishm = self._iflat, self.shm._last.value + to_update = rfn.structured_to_unstructured( + self.shm._array[iflat:ishm][fields] + ) + + # print(to_update) + self.flat[iflat:ishm][:] = to_update + profiler('updated ustruct OHLC data') + + y_flat = self.flat[:ishm] + x_flat = self.x_basis[:ishm] + + self._iflat = ishm + + y = y_flat.reshape(-1) + x = x_flat.reshape(-1) + profiler('flattened ustruct OHLC data') + + y_iv_flat = y_flat[ivl:ivr] + x_iv_flat = x_flat[ivl:ivr] + + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') curve.update_from_array( x, y, x_iv=x_iv, - y_iv=y_iv, - view_range=None, # hack + y_iv=y_iv, + view_range=(ivl, ivr), # hack profiler=profiler, ) profiler('updated ds curve') @@ -477,7 +504,7 @@ def update_graphics( # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - graphics.prepareGeometryChange() + # graphics.prepareGeometryChange() graphics.update() if ( @@ -632,7 +659,7 @@ def render( ''' # do full source data render to path - last_read = ( + ( xfirst, xlast, array, ivl, ivr, in_view, ) = self.last_read From df78e9ba964b319dd0f367c19cf560528074c142 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Apr 2022 15:15:00 -0400 Subject: [PATCH 011/113] Delegate graphics cycle max/min to chart/flows --- piker/ui/_display.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index fda3fb042..0e03395ef 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -96,28 +96,17 @@ def chart_maxmin( Compute max and min datums "in view" for range limits. ''' - array = ohlcv_shm.array - ifirst = array[0]['index'] - last_bars_range = chart.bars_range() - l, lbar, rbar, r = last_bars_range - in_view = array[lbar - ifirst:rbar - ifirst + 1] + out = chart.maxmin() - if not in_view.size: - log.warning('Resetting chart to data') - chart.default_view() + if out is None: return (last_bars_range, 0, 0, 0) - mx, mn = ( - np.nanmax(in_view['high']), - np.nanmin(in_view['low'],) - ) + mn, mx = out mx_vlm_in_view = 0 if vlm_chart: - mx_vlm_in_view = np.max( - in_view['volume'] - ) + _, mx_vlm_in_view = vlm_chart.maxmin() return ( last_bars_range, From 2af4050e5e3615204cf42724faf46458df1bab95 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Apr 2022 15:47:24 -0400 Subject: [PATCH 012/113] Remove `._set_yrange()` handler from x-range-change signal --- piker/ui/_interaction.py | 56 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4b3bbb45d..ca49a35df 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -492,7 +492,10 @@ def wheelEvent( log.debug("Max zoom bruh...") return - # if ev.delta() < 0 and vl >= len(chart._flows[chart.name].shm.array) + 666: + # if ( + # ev.delta() < 0 + # and vl >= len(chart._flows[chart.name].shm.array) + 666 + # ): # log.debug("Min zoom bruh...") # return @@ -748,6 +751,7 @@ def _set_yrange( ''' profiler = pg.debug.Profiler( + msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), gt=ms_slower_then, delayed=True, @@ -815,6 +819,7 @@ def _set_yrange( if yrange is None: log.warning(f'No yrange provided for {self.name}!?') + print(f"WTF NO YRANGE {self.name}") return ylow, yhigh = yrange @@ -851,11 +856,6 @@ def enable_auto_yrange( if src_vb is None: src_vb = self - # such that when a linked chart changes its range - # this local view is also automatically changed and - # resized to data. - src_vb.sigXRangeChanged.connect(self._set_yrange) - # splitter(s) resizing src_vb.sigResized.connect(self._set_yrange) @@ -876,11 +876,6 @@ def disable_auto_yrange( self, ) -> None: - # self._chart._static_yrange = 'axis' - - self.sigXRangeChanged.disconnect( - self._set_yrange, - ) self.sigResized.disconnect( self._set_yrange, ) @@ -911,18 +906,20 @@ def x_uppx(self) -> float: def maybe_downsample_graphics(self): - profiler = pg.debug.Profiler( - disabled=not pg_profile_enabled(), - gt=3, - ) - uppx = self.x_uppx() if not ( # we probably want to drop this once we are "drawing in # view" for downsampled flows.. - uppx and uppx > 16 + uppx and uppx > 6 and self._ic is not None ): + profiler = pg.debug.Profiler( + msg=f'ChartView.maybe_downsample_graphics() for {self.name}', + disabled=not pg_profile_enabled(), + # delayed=True, + gt=3, + # gt=ms_slower_then, + ) # TODO: a faster single-loop-iterator way of doing this XD chart = self._chart @@ -931,7 +928,12 @@ def maybe_downsample_graphics(self): for chart_name, chart in plots.items(): for name, flow in chart._flows.items(): - if not flow.render: + if ( + not flow.render + + # XXX: super important to be aware of this. + # or not flow.graphics.isVisible() + ): continue graphics = flow.graphics @@ -947,13 +949,17 @@ def maybe_downsample_graphics(self): use_vr=use_vr, # gets passed down into graphics obj - profiler=profiler, + # profiler=profiler, ) profiler(f'range change updated {chart_name}:{name}') - else: - # don't bother updating since we're zoomed out bigly and - # in a pan-interaction, in which case we shouldn't be - # doing view-range based rendering (at least not yet). - # print(f'{uppx} exiting early!') - profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') + + profiler.finish() + # else: + # # don't bother updating since we're zoomed out bigly and + # # in a pan-interaction, in which case we shouldn't be + # # doing view-range based rendering (at least not yet). + # # print(f'{uppx} exiting early!') + # profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') + + # profiler.finish() From c94c53286b6e6261742d0c89b2fa31ede6a71314 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 13:59:20 -0400 Subject: [PATCH 013/113] `FastAppendCurve`: Only render in-view data if possible More or less this improves update latency like mad. Only draw data in view and avoid full path regen as much as possible within a given (down)sampling setting. We now support append path updates with in-view data and the *SPECIAL CAVEAT* is that we avoid redrawing the whole curve **only when** we calc an `append_length <= 1` **even if the view range changed**. XXX: this should change in the future probably such that the caller graphics update code can pass a flag which says whether or not to do a full redraw based on it knowing where it's an interaction based view-range change or a flow update change which doesn't require a full path re-render. --- piker/ui/_curve.py | 116 +++++++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 46 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 47132d7ce..8c3c329fd 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -138,6 +138,7 @@ def __init__( # brutaaalll, see comments within.. self._y = self.yData = y self._x = self.xData = x + self._vr: Optional[tuple] = None self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -287,6 +288,17 @@ def update_from_array( istart, istop = self._xrange else: self._xrange = istart, istop = x[0], x[-1] + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length = int(istart - x[0]) + append_length = int(x[-1] - istop) + + # this is the diff-mode, "data"-rendered index + # tracking var.. + self._xrange = x[0], x[-1] + # print(f"xrange: {self._xrange}") # XXX: lol brutal, the internals of `CurvePoint` (inherited by @@ -295,37 +307,36 @@ def update_from_array( # self.yData = y # self._x, self._y = x, y - if view_range: - profiler(f'view range slice {view_range}') - # downsampling incremental state checking uppx = self.x_uppx() px_width = self.px_width() uppx_diff = (uppx - self._last_uppx) + new_sample_rate = False should_ds = False + showing_src_data = self._in_ds should_redraw = False # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. - if view_range and not self._in_ds: + if ( + view_range + # and not self._in_ds + # and not prepend_length > 0 + ): # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index x_out, y_out = x_iv[:-1], y_iv[:-1] + profiler(f'view range slice {view_range}') - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - # TODO: numba this bish - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out - ) - profiler('generated step arrays') + if ( + view_range != self._vr + and append_length > 1 + ): + should_redraw = True - should_redraw = True - profiler('sliced in-view array history') + self._vr = view_range # x_last = x_iv[-1] # y_last = y_iv[-1] @@ -335,7 +346,15 @@ def update_from_array( # flip_cache = True else: - self._xrange = x[0], x[-1] + # if ( + # not view_range + # or self._in_ds + # ): + # by default we only pull data up to the last (current) index + x_out, y_out = x[:-1], y[:-1] + + if prepend_length > 0: + should_redraw = True # check for downsampling conditions if ( @@ -350,6 +369,8 @@ def update_from_array( f'{self._name} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False should_ds = True elif ( @@ -360,49 +381,47 @@ def update_from_array( # source data so we clear our path data in prep # to generate a new one from original source data. should_redraw = True + new_sample_rate = True should_ds = False - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - prepend_length = int(istart - x[0]) - append_length = int(x[-1] - istop) + showing_src_data = True # no_path_yet = self.path is None if ( self.path is None or should_redraw - or should_ds + or new_sample_rate or prepend_length > 0 ): - if ( - not view_range - or self._in_ds - ): - # by default we only pull data up to the last (current) index - x_out, y_out = x[:-1], y[:-1] - - # step mode: draw flat top discrete "step" - # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out, - ) - # TODO: numba this bish - profiler('generated step arrays') + # if ( + # not view_range + # or self._in_ds + # ): + # # by default we only pull data up to the last (current) index + # x_out, y_out = x[:-1], y[:-1] + + # step mode: draw flat top discrete "step" + # over the index space for each datum. + if self._step_mode: + x_out, y_out = step_path_arrays_from_1d( + x_out, + y_out, + ) + # TODO: numba this bish + profiler('generated step arrays') if should_redraw: - profiler('path reversion to non-ds') if self.path: + # print(f'CLEARING PATH {self._name}') self.path.clear() if self.fast_path: self.fast_path.clear() - if should_redraw and not should_ds: - if self._in_ds: - log.info(f'DEDOWN -> {self._name}') + profiler('cleared paths due to `should_redraw` set') + + if new_sample_rate and showing_src_data: + # if self._in_ds: + log.info(f'DEDOWN -> {self._name}') self._in_ds = False @@ -423,7 +442,12 @@ def update_from_array( finiteCheck=False, path=self.path, ) - profiler('generated fresh path') + profiler( + 'generated fresh path\n' + f'should_redraw: {should_redraw}\n' + f'should_ds: {should_ds}\n' + f'new_sample_rate: {new_sample_rate}\n' + ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') # reserve mem allocs see: @@ -455,7 +479,7 @@ def update_from_array( elif ( append_length > 0 - and not view_range + # and not view_range ): new_x = x[-append_length - 2:-1] new_y = y[-append_length - 2:-1] @@ -696,7 +720,7 @@ def paint( if path: p.drawPath(path) - profiler('.drawPath(path)') + profiler(f'.drawPath(path): {path.capacity()}') fp = self.fast_path if fp: From af6aad4e9c612c144b5304fbf68b666ce32ace05 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 14:06:48 -0400 Subject: [PATCH 014/113] If a sample stream is already ded, just warn --- piker/data/_sampling.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 8bc677cf7..466ef0e71 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -168,7 +168,12 @@ async def broadcast( log.error( f'{stream._ctx.chan.uid} dropped connection' ) - subs.remove(stream) + try: + subs.remove(stream) + except ValueError: + log.warning( + f'{stream._ctx.chan.uid} sub already removed!?' + ) @tractor.context From 64206543cda6ecd370b2c94e4b880c198dcb3abc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 19:01:37 -0400 Subject: [PATCH 015/113] Put mxmn profile mapping at end of method --- piker/ui/_chart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8aa100911..cecbbff5c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1283,7 +1283,7 @@ def maxmin( flow is None ): log.error(f"flow {flow_key} doesn't exist in chart {self.name} !?") - res = 0, 0 + key = res = 0, 0 else: first, l, lbar, rbar, r, last = bars_range or flow.datums_range() @@ -1291,11 +1291,11 @@ def maxmin( key = round(lbar), round(rbar) res = flow.maxmin(*key) - profiler(f'yrange mxmn: {key} -> {res}') if res == (None, None): log.error( f"{flow_key} no mxmn for bars_range => {key} !?" ) res = 0, 0 + profiler(f'yrange mxmn: {key} -> {res}') return res From db727910bebe8e43449fefc5fa8a3b9b8650f385 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 19:02:22 -0400 Subject: [PATCH 016/113] Always use coord cache, add naive view range diffing logic --- piker/ui/_curve.py | 56 ++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8c3c329fd..4b87d1176 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -183,16 +183,16 @@ def __init__( # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - if step_mode: - # don't enable caching by default for the case where the - # only thing drawn is the "last" line segment which can - # have a weird artifact where it won't be fully drawn to its - # endpoint (something we saw on trade rate curves) - self.setCacheMode( - QGraphicsItem.DeviceCoordinateCache - ) + # if step_mode: + # don't enable caching by default for the case where the + # only thing drawn is the "last" line segment which can + # have a weird artifact where it won't be fully drawn to its + # endpoint (something we saw on trade rate curves) + self.setCacheMode( + QGraphicsItem.DeviceCoordinateCache + ) - self.update() + # self.update() # TODO: probably stick this in a new parent # type which will contain our own version of @@ -313,7 +313,7 @@ def update_from_array( uppx_diff = (uppx - self._last_uppx) new_sample_rate = False - should_ds = False + should_ds = self._in_ds showing_src_data = self._in_ds should_redraw = False @@ -330,11 +330,27 @@ def update_from_array( x_out, y_out = x_iv[:-1], y_iv[:-1] profiler(f'view range slice {view_range}') + ivl, ivr = view_range + + probably_zoom_change = False + last_vr = self._vr + if last_vr: + livl, livr = last_vr + if ( + ivl < livl + or (ivr - livr) > 2 + ): + probably_zoom_change = True + if ( - view_range != self._vr - and append_length > 1 + view_range != last_vr + and ( + append_length > 1 + or probably_zoom_change + ) ): should_redraw = True + # print("REDRAWING BRUH") self._vr = view_range @@ -371,6 +387,7 @@ def update_from_array( self._last_uppx = uppx new_sample_rate = True showing_src_data = False + should_redraw = True should_ds = True elif ( @@ -504,13 +521,14 @@ def update_from_array( f'diffed array input, append_length={append_length}' ) - if should_ds: - new_x, new_y = self.downsample( - new_x, - new_y, - **should_ds, - ) - profiler(f'fast path downsample redraw={should_ds}') + # if should_ds: + # new_x, new_y = self.downsample( + # new_x, + # new_y, + # px_width, + # uppx, + # ) + # profiler(f'fast path downsample redraw={should_ds}') append_path = pg.functions.arrayToQPath( new_x, From aee44fed46351791de0bb50ef6cc3ef0b3b144c5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 22 Apr 2022 23:02:02 -0400 Subject: [PATCH 017/113] Right, handle the case where the shm prepend history isn't full XD --- piker/ui/_flows.py | 143 +++++++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 56 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 20ffc510f..a8f7d0a59 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -37,7 +37,7 @@ from ..data._sharedmem import ( ShmArray, - open_shm_array, + # open_shm_array, ) from .._profile import pg_profile_enabled, ms_slower_then from ._ohlc import ( @@ -48,7 +48,7 @@ FastAppendCurve, ) from ._compression import ( - ohlc_flatten, + # ohlc_flatten, ds_m4, ) from ..log import get_logger @@ -103,15 +103,15 @@ def rowarr_to_path( ) -def ohlc_flat_view( +def mk_ohlc_flat_copy( ohlc_shm: ShmArray, # XXX: we bind this in currently.. - x_basis: np.ndarray, + # x_basis: np.ndarray, # vr: Optional[slice] = None, -) -> np.ndarray: +) -> tuple[np.ndarray, np.ndarray]: ''' Return flattened-non-copy view into an OHLC shm array. @@ -127,8 +127,8 @@ def ohlc_flat_view( ) # breakpoint() y = unstructured.flatten() - x = x_basis[:y.size] - return x, y + # x = x_basis[:y.size] + return y class Flow(msgspec.Struct): # , frozen=True): @@ -151,7 +151,8 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop flat: Optional[ShmArray] = None x_basis: Optional[np.ndarray] = None - _iflat: int = 0 + _iflat_last: int = 0 + _iflat_first: int = 0 _last_uppx: float = 0 _in_ds: bool = False @@ -344,7 +345,6 @@ def update_graphics( graphics = self.graphics if isinstance(graphics, BarItems): - fields = ['open', 'high', 'low', 'close'] # if no source data renderer exists create one. r = self._src_r if not r: @@ -356,49 +356,11 @@ def update_graphics( last_read=read, ) - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - - # flat = self.flat = self.shm.unstruct_view(fields) - self.flat = self.shm.ustruct(fields) - self._iflat = self.shm._last.value - - # import pdbpp - # pdbpp.set_trace() - # assert len(flat._array) == len(self.shm._array[fields]) - - x = self.x_basis = ( - np.broadcast_to( - shm._array['index'][:, None], - ( - shm._array.size, - # 4, # only ohlc - self.flat.shape[1], - ), - ) + np.array([-0.5, 0, 0, 0.5]) - ) - - # fshm = self.flat = open_shm_array( - # f'{self.name}_flat', - # dtype=flattened.dtype, - # size=flattened.size, - # ) - # fshm.push(flattened) - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.flat._first.value - ds_curve_r = Renderer( flow=self, # just swap in the flat view - data_t=lambda array: self.flat.array, - # data_t=partial( - # ohlc_flat_view, - # self.shm, - # ), + # data_t=lambda array: self.flat.array, last_read=read, draw_path=partial( rowarr_to_path, @@ -435,12 +397,14 @@ def update_graphics( should_line and uppx < x_gt ): + print('FLIPPING TO BARS') should_line = False elif ( not should_line and uppx >= x_gt ): + print('FLIPPING TO LINE') should_line = True profiler(f'ds logic complete line={should_line}') @@ -448,32 +412,98 @@ def update_graphics( # do graphics updates if should_line: + fields = ['open', 'high', 'low', 'close'] + if self.flat is None: + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + + # flat = self.flat = self.shm.unstruct_view(fields) + self.flat = self.shm.ustruct(fields) + first = self._iflat_first = self.shm._first.value + last = self._iflat_last = self.shm._last.value + + # write pushed data to flattened copy + self.flat[first:last] = rfn.structured_to_unstructured( + self.shm.array[fields] + ) + + # generate an flat-interpolated x-domain + self.x_basis = ( + np.broadcast_to( + shm._array['index'][:, None], + ( + shm._array.size, + # 4, # only ohlc + self.flat.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + assert self.flat.any() + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.flat._first.value # update flatted ohlc copy - iflat, ishm = self._iflat, self.shm._last.value + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value + ) + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + self.flat[ + ishm_first:iflat_first + ] = rfn.structured_to_unstructured( + self.shm.array[fields][:iflat_first] + ) + self._iflat_first = ishm_first + + # # flat = self.flat = self.shm.unstruct_view(fields) + # self.flat = self.shm.ustruct(fields) + # # self._iflat_last = self.shm._last.value + + # # self._iflat_first = self.shm._first.value + # # do an update for the most recent prepend + # # index + # iflat = ishm_first + to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm][fields] + self.shm._array[iflat:ishm_last][fields] ) - # print(to_update) - self.flat[iflat:ishm][:] = to_update + self.flat[iflat:ishm_last][:] = to_update profiler('updated ustruct OHLC data') - y_flat = self.flat[:ishm] - x_flat = self.x_basis[:ishm] + # slice out up-to-last step contents + y_flat = self.flat[ishm_first:ishm_last] + x_flat = self.x_basis[ishm_first:ishm_last] - self._iflat = ishm + # update local last-index tracking + self._iflat_last = ishm_last + # reshape to 1d for graphics rendering y = y_flat.reshape(-1) x = x_flat.reshape(-1) profiler('flattened ustruct OHLC data') + # do all the same for only in-view data y_iv_flat = y_flat[ivl:ivr] x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') + # legacy full-recompute-everytime method # x, y = ohlc_flatten(array) # x_iv, y_iv = ohlc_flatten(in_view) # profiler('flattened OHLC data') @@ -486,6 +516,7 @@ def update_graphics( view_range=(ivl, ivr), # hack profiler=profiler, ) + curve.show() profiler('updated ds curve') else: From 69282a99246f50c36ebb22f89c03261c067fa46e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 15:33:40 -0400 Subject: [PATCH 018/113] Handle null output case for vlm chart mxmn --- piker/ui/_display.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 0e03395ef..b82d1253f 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -106,7 +106,9 @@ def chart_maxmin( mx_vlm_in_view = 0 if vlm_chart: - _, mx_vlm_in_view = vlm_chart.maxmin() + out = vlm_chart.maxmin() + if out: + _, mx_vlm_in_view = out return ( last_bars_range, From 64c6287cd1f046e9c4c67526bf4ec0169558515f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 17:22:02 -0400 Subject: [PATCH 019/113] Always set coords cache on curves --- piker/ui/_curve.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4b87d1176..0befe4541 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -188,11 +188,7 @@ def __init__( # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its # endpoint (something we saw on trade rate curves) - self.setCacheMode( - QGraphicsItem.DeviceCoordinateCache - ) - - # self.update() + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # TODO: probably stick this in a new parent # type which will contain our own version of @@ -423,6 +419,9 @@ def update_from_array( x_out, y_out, ) + # self.disable_cache() + # flip_cache = True + # TODO: numba this bish profiler('generated step arrays') @@ -514,6 +513,9 @@ def update_from_array( new_x = new_x[1:] new_y = new_y[1:] + # self.disable_cache() + # flip_cache = True + profiler('generated step data') else: From b97ec38baff4f8ca5db1bdb13cdf5885ceda4316 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 23 Apr 2022 17:22:28 -0400 Subject: [PATCH 020/113] Always maybe render graphics Since we have in-view style rendering working for all curve types (finally) we can avoid the guard for low uppx levels and without losing interaction speed. Further don't delay the profiler so that the nested method calls correctly report upward - which wasn't working likely due to some kinda GC collection related issue. --- piker/ui/_interaction.py | 84 +++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index ca49a35df..943f33709 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -20,7 +20,6 @@ """ from __future__ import annotations from contextlib import asynccontextmanager -# import itertools import time from typing import Optional, Callable @@ -907,54 +906,59 @@ def x_uppx(self) -> float: def maybe_downsample_graphics(self): uppx = self.x_uppx() - if not ( - # we probably want to drop this once we are "drawing in - # view" for downsampled flows.. - uppx and uppx > 6 - and self._ic is not None - ): - profiler = pg.debug.Profiler( - msg=f'ChartView.maybe_downsample_graphics() for {self.name}', - disabled=not pg_profile_enabled(), - # delayed=True, - gt=3, - # gt=ms_slower_then, - ) + # if not ( + # # we probably want to drop this once we are "drawing in + # # view" for downsampled flows.. + # uppx and uppx > 6 + # and self._ic is not None + # ): + profiler = pg.debug.Profiler( + msg=f'ChartView.maybe_downsample_graphics() for {self.name}', + disabled=not pg_profile_enabled(), - # TODO: a faster single-loop-iterator way of doing this XD - chart = self._chart - linked = self.linkedsplits - plots = linked.subplots | {chart.name: chart} - for chart_name, chart in plots.items(): - for name, flow in chart._flows.items(): + # XXX: important to avoid not seeing underlying + # ``.update_graphics_from_flow()`` nested profiling likely + # due to the way delaying works and garbage collection of + # the profiler in the delegated method calls. + delayed=False, + # gt=3, + # gt=ms_slower_then, + ) - if ( - not flow.render + # TODO: a faster single-loop-iterator way of doing this XD + chart = self._chart + linked = self.linkedsplits + plots = linked.subplots | {chart.name: chart} + for chart_name, chart in plots.items(): + for name, flow in chart._flows.items(): - # XXX: super important to be aware of this. - # or not flow.graphics.isVisible() - ): - continue + if ( + not flow.render - graphics = flow.graphics + # XXX: super important to be aware of this. + # or not flow.graphics.isVisible() + ): + continue - use_vr = False - if isinstance(graphics, BarItems): - use_vr = True + graphics = flow.graphics - # pass in no array which will read and render from the last - # passed array (normally provided by the display loop.) - chart.update_graphics_from_flow( - name, - use_vr=use_vr, + # use_vr = False + # if isinstance(graphics, BarItems): + # use_vr = True - # gets passed down into graphics obj - # profiler=profiler, - ) + # pass in no array which will read and render from the last + # passed array (normally provided by the display loop.) + chart.update_graphics_from_flow( + name, + use_vr=True, - profiler(f'range change updated {chart_name}:{name}') + # gets passed down into graphics obj + # profiler=profiler, + ) - profiler.finish() + profiler(f'range change updated {chart_name}:{name}') + + profiler.finish() # else: # # don't bother updating since we're zoomed out bigly and # # in a pan-interaction, in which case we shouldn't be From b2b31b8f841799d843eb991d35dc4eff02703a76 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 12:33:25 -0400 Subject: [PATCH 021/113] WIP incrementally update step array format --- piker/ui/_curve.py | 114 +++++++++++++++++------------- piker/ui/_flows.py | 168 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 214 insertions(+), 68 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 0befe4541..e28035492 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -58,36 +58,55 @@ def step_path_arrays_from_1d( ''' y_out = y.copy() x_out = x.copy() - x2 = np.empty( - # the data + 2 endpoints on either end for - # "termination of the path". - (len(x) + 1, 2), - # we want to align with OHLC or other sampling style - # bars likely so we need fractinal values - dtype=float, - ) - x2[0] = x[0] - 0.5 - x2[1] = x[0] + 0.5 - x2[1:] = x[:, np.newaxis] + 0.5 + + # x2 = np.empty( + # # the data + 2 endpoints on either end for + # # "termination of the path". + # (len(x) + 1, 2), + # # we want to align with OHLC or other sampling style + # # bars likely so we need fractinal values + # dtype=float, + # ) + + x2 = np.broadcast_to( + x[:, None], + ( + x_out.size, + # 4, # only ohlc + 2, + ), + ) + np.array([-0.5, 0.5]) + + # x2[0] = x[0] - 0.5 + # x2[1] = x[0] + 0.5 + # x2[0, 0] = x[0] - 0.5 + # x2[0, 1] = x[0] + 0.5 + # x2[1:] = x[:, np.newaxis] + 0.5 + # import pdbpp + # pdbpp.set_trace() # flatten to 1-d - x_out = x2.reshape(x2.size) + # x_out = x2.reshape(x2.size) + x_out = x2 # we create a 1d with 2 extra indexes to # hold the start and (current) end value for the steps # on either end y2 = np.empty((len(y), 2), dtype=y.dtype) y2[:] = y[:, np.newaxis] + y2[-1] = 0 + + y_out = y2 - y_out = np.empty( - 2*len(y) + 2, - dtype=y.dtype - ) +# y_out = np.empty( +# 2*len(y) + 2, +# dtype=y.dtype +# ) # flatten and set 0 endpoints - y_out[1:-1] = y2.reshape(y2.size) - y_out[0] = 0 - y_out[-1] = 0 + # y_out[1:-1] = y2.reshape(y2.size) + # y_out[0] = 0 + # y_out[-1] = 0 if not include_endpoints: return x_out[:-1], y_out[:-1] @@ -414,16 +433,16 @@ def update_from_array( # step mode: draw flat top discrete "step" # over the index space for each datum. - if self._step_mode: - x_out, y_out = step_path_arrays_from_1d( - x_out, - y_out, - ) - # self.disable_cache() - # flip_cache = True + # if self._step_mode: + # x_out, y_out = step_path_arrays_from_1d( + # x_out, + # y_out, + # ) + # # self.disable_cache() + # # flip_cache = True - # TODO: numba this bish - profiler('generated step arrays') + # # TODO: numba this bish + # profiler('generated step arrays') if should_redraw: if self.path: @@ -501,27 +520,26 @@ def update_from_array( new_y = y[-append_length - 2:-1] profiler('sliced append path') - if self._step_mode: - new_x, new_y = step_path_arrays_from_1d( - new_x, - new_y, - ) - # [1:] since we don't need the vertical line normally at - # the beginning of the step curve taking the first (x, - # y) poing down to the x-axis **because** this is an - # appended path graphic. - new_x = new_x[1:] - new_y = new_y[1:] + # if self._step_mode: + # new_x, new_y = step_path_arrays_from_1d( + # new_x, + # new_y, + # ) + # # [1:] since we don't need the vertical line normally at + # # the beginning of the step curve taking the first (x, + # # y) poing down to the x-axis **because** this is an + # # appended path graphic. + # new_x = new_x[1:] + # new_y = new_y[1:] - # self.disable_cache() - # flip_cache = True + # # self.disable_cache() + # # flip_cache = True - profiler('generated step data') + # profiler('generated step data') - else: - profiler( - f'diffed array input, append_length={append_length}' - ) + profiler( + f'diffed array input, append_length={append_length}' + ) # if should_ds: # new_x, new_y = self.downsample( @@ -655,6 +673,10 @@ def clear(self): # self.disable_cache() # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + def reset_cache(self) -> None: + self.disable_cache() + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + def disable_cache(self) -> None: ''' Disable the use of the pixel coordinate cache and trigger a geo event. diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a8f7d0a59..b150a2d1b 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -46,6 +46,7 @@ ) from ._curve import ( FastAppendCurve, + step_path_arrays_from_1d, ) from ._compression import ( # ohlc_flatten, @@ -149,8 +150,8 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop - flat: Optional[ShmArray] = None - x_basis: Optional[np.ndarray] = None + gy: Optional[ShmArray] = None + gx: Optional[np.ndarray] = None _iflat_last: int = 0 _iflat_first: int = 0 @@ -360,7 +361,7 @@ def update_graphics( flow=self, # just swap in the flat view - # data_t=lambda array: self.flat.array, + # data_t=lambda array: self.gy.array, last_read=read, draw_path=partial( rowarr_to_path, @@ -413,37 +414,37 @@ def update_graphics( if should_line: fields = ['open', 'high', 'low', 'close'] - if self.flat is None: + if self.gy is None: # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm - # flat = self.flat = self.shm.unstruct_view(fields) - self.flat = self.shm.ustruct(fields) + # flat = self.gy = self.shm.unstruct_view(fields) + self.gy = self.shm.ustruct(fields) first = self._iflat_first = self.shm._first.value last = self._iflat_last = self.shm._last.value # write pushed data to flattened copy - self.flat[first:last] = rfn.structured_to_unstructured( + self.gy[first:last] = rfn.structured_to_unstructured( self.shm.array[fields] ) # generate an flat-interpolated x-domain - self.x_basis = ( + self.gx = ( np.broadcast_to( shm._array['index'][:, None], ( shm._array.size, # 4, # only ohlc - self.flat.shape[1], + self.gy.shape[1], ), ) + np.array([-0.5, 0, 0, 0.5]) ) - assert self.flat.any() + assert self.gy.any() # print(f'unstruct diff: {time.time() - start}') # profiler('read unstr view bars to line') - # start = self.flat._first.value + # start = self.gy._first.value # update flatted ohlc copy ( iflat_first, @@ -461,15 +462,15 @@ def update_graphics( if iflat_first != ishm_first: # write newly prepended data to flattened copy - self.flat[ + self.gy[ ishm_first:iflat_first ] = rfn.structured_to_unstructured( self.shm.array[fields][:iflat_first] ) self._iflat_first = ishm_first - # # flat = self.flat = self.shm.unstruct_view(fields) - # self.flat = self.shm.ustruct(fields) + # # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) # # self._iflat_last = self.shm._last.value # # self._iflat_first = self.shm._first.value @@ -481,12 +482,12 @@ def update_graphics( self.shm._array[iflat:ishm_last][fields] ) - self.flat[iflat:ishm_last][:] = to_update + self.gy[iflat:ishm_last][:] = to_update profiler('updated ustruct OHLC data') # slice out up-to-last step contents - y_flat = self.flat[ishm_first:ishm_last] - x_flat = self.x_basis[ishm_first:ishm_last] + y_flat = self.gy[ishm_first:ishm_last] + x_flat = self.gx[ishm_first:ishm_last] # update local last-index tracking self._iflat_last = ishm_last @@ -577,16 +578,139 @@ def update_graphics( # graphics.draw_last(last) else: - # ``FastAppendCurve`` case: array_key = array_key or self.name + # ``FastAppendCurve`` case: + if graphics._step_mode and self.gy is None: + + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + + # fields = ['index', array_key] + i = shm._array['index'] + out = shm._array[array_key] + + self.gx, self.gy = step_path_arrays_from_1d(i, out) + + # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) + # first = self._iflat_first = self.shm._first.value + # last = self._iflat_last = self.shm._last.value + + # # write pushed data to flattened copy + # self.gy[first:last] = rfn.structured_to_unstructured( + # self.shm.array[fields] + # ) + + # # generate an flat-interpolated x-domain + # self.gx = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # ( + # shm._array.size, + # # 4, # only ohlc + # self.gy.shape[1], + # ), + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) + # assert self.gy.any() + + # print(f'unstruct diff: {time.time() - start}') + # profiler('read unstr view bars to line') + # start = self.gy._first.value + # update flatted ohlc copy + + if graphics._step_mode: + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value + ) + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + _gx, self.gy[ + ishm_first:iflat_first + ] = step_path_arrays_from_1d( + self.shm.array['index'][:iflat_first], + self.shm.array[array_key][:iflat_first], + ) + self._iflat_first = ishm_first + # # flat = self.gy = self.shm.unstruct_view(fields) + # self.gy = self.shm.ustruct(fields) + # # self._iflat_last = self.shm._last.value + + # # self._iflat_first = self.shm._first.value + # # do an update for the most recent prepend + # # index + # iflat = ishm_first + if iflat != ishm_last: + _x, to_update = step_path_arrays_from_1d( + self.shm._array[iflat:ishm_last]['index'], + self.shm._array[iflat:ishm_last][array_key], + ) + + # to_update = rfn.structured_to_unstructured( + # self.shm._array[iflat:ishm_last][fields] + # ) + + # import pdbpp + # pdbpp.set_trace() + self.gy[iflat:ishm_last-1] = to_update + self.gy[-1] = 0 + print(f'updating step curve {to_update}') + profiler('updated step curve data') + + # slice out up-to-last step contents + x_step = self.gx[ishm_first:ishm_last] + x = x_step.reshape(-1) + y_step = self.gy[ishm_first:ishm_last] + y = y_step.reshape(-1) + profiler('sliced step data') + + # update local last-index tracking + self._iflat_last = ishm_last + + # reshape to 1d for graphics rendering + # y = y_flat.reshape(-1) + # x = x_flat.reshape(-1) + + # do all the same for only in-view data + y_iv = y_step[ivl:ivr].reshape(-1) + x_iv = x_step[ivl:ivr].reshape(-1) + # y_iv = y_iv_flat.reshape(-1) + # x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # legacy full-recompute-everytime method + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') + graphics.reset_cache() + + else: + x = array['index'] + y = array[array_key] + x_iv = in_view['index'] + y_iv = in_view[array_key] + graphics.update_from_array( - x=array['index'], - y=array[array_key], + x=x, + y=y, + + x_iv=x_iv, + y_iv=y_iv, - x_iv=in_view['index'], - y_iv=in_view[array_key], view_range=(ivl, ivr) if use_vr else None, **kwargs From 82b2d2ee3a4be377246330c2dadb6e0c0ccbf755 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 5 Apr 2022 14:54:13 -0400 Subject: [PATCH 022/113] Hipshot, use uppx to drive theoretical px w --- piker/ui/_compression.py | 44 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index adb422515..a6102eab1 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -181,6 +181,7 @@ def ds_m4( # in display-device-local pixel units. px_width: int, uppx: Optional[float] = None, + xrange: Optional[float] = None, log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: @@ -212,6 +213,7 @@ def ds_m4( # as the units-per-px (uppx) get's large. if log_scale: assert uppx, 'You must provide a `uppx` value to use log scaling!' + # uppx = uppx * math.log(uppx, 2) # scaler = 2**7 / (1 + math.log(uppx, 2)) scaler = round( @@ -223,37 +225,63 @@ def ds_m4( 1 ) ) - px_width *= scaler + # px_width *= scaler + + # else: + # px_width *= 16 assert px_width > 1 # width of screen in pxs? + assert uppx > 0 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, # do we care though since we can always just pre-slice the # input? x_start = x[0] # x value start/lowest in domain - x_end = x[-1] # x end value/highest in domain + + if xrange is None: + x_end = x[-1] # x end value/highest in domain + xrange = (x_end - x_start) # XXX: always round up on the input pixels - px_width = math.ceil(px_width) + # lnx = len(x) + # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) - x_range = x_end - x_start + pxw = math.ceil(xrange / uppx) + px_width = math.ceil(px_width) # ratio of indexed x-value to width of raster in pixels. # this is more or less, uppx: units-per-pixel. - w = x_range / float(px_width) + # w = xrange / float(px_width) + # uppx = uppx * math.log(uppx, 2) + w2 = px_width / uppx + + # scale up the width as the uppx get's large + w = uppx# * math.log(uppx, 666) # ensure we make more then enough # frames (windows) for the output pixel - frames = px_width + frames = pxw # if we have more and then exact integer's # (uniform quotient output) worth of datum-domain-points # per windows-frame, add one more window to ensure # we have room for all output down-samples. - pts_per_pixel, r = divmod(len(x), frames) + pts_per_pixel, r = divmod(xrange, frames) if r: - frames += 1 + while r: + frames += 1 + pts_per_pixel, r = divmod(xrange, frames) + + print( + f'uppx: {uppx}\n' + f'xrange: {xrange}\n' + f'px_width: {px_width}\n' + f'pxw: {pxw}\n' + f'WTF w:{w}, w2:{w2}\n' + f'frames: {frames}\n' + ) + assert frames >= (xrange / uppx) # call into ``numba`` nb, i_win, y_out = _m4( From ba0ba346ec951324392fccd00422b46db8bb2f5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:08:16 -0400 Subject: [PATCH 023/113] Drop log scaling support since uppx driven scaling seems way faster/better --- piker/ui/_compression.py | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index a6102eab1..5e8b759a4 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -162,7 +162,7 @@ def ohlc_to_m4_line( flat, px_width=px_width, uppx=uppx, - log_scale=bool(uppx) + # log_scale=bool(uppx) ) x = np.broadcast_to(x[:, None], y.shape) x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -182,7 +182,7 @@ def ds_m4( px_width: int, uppx: Optional[float] = None, xrange: Optional[float] = None, - log_scale: bool = True, + # log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: ''' @@ -211,27 +211,27 @@ def ds_m4( # optionally log-scale down the "supposed pxs on screen" # as the units-per-px (uppx) get's large. - if log_scale: - assert uppx, 'You must provide a `uppx` value to use log scaling!' - # uppx = uppx * math.log(uppx, 2) - - # scaler = 2**7 / (1 + math.log(uppx, 2)) - scaler = round( - max( - # NOTE: found that a 16x px width brought greater - # detail, likely due to dpi scaling? - # px_width=px_width * 16, - 2**7 / (1 + math.log(uppx, 2)), - 1 - ) - ) - # px_width *= scaler + # if log_scale: + # assert uppx, 'You must provide a `uppx` value to use log scaling!' + # # uppx = uppx * math.log(uppx, 2) + + # # scaler = 2**7 / (1 + math.log(uppx, 2)) + # scaler = round( + # max( + # # NOTE: found that a 16x px width brought greater + # # detail, likely due to dpi scaling? + # # px_width=px_width * 16, + # 2**7 / (1 + math.log(uppx, 2)), + # 1 + # ) + # ) + # px_width *= scaler # else: # px_width *= 16 - assert px_width > 1 # width of screen in pxs? - assert uppx > 0 + # should never get called unless actually needed + assert px_width > 1 and uppx > 0 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, @@ -248,16 +248,16 @@ def ds_m4( # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) pxw = math.ceil(xrange / uppx) - px_width = math.ceil(px_width) + # px_width = math.ceil(px_width) # ratio of indexed x-value to width of raster in pixels. # this is more or less, uppx: units-per-pixel. # w = xrange / float(px_width) # uppx = uppx * math.log(uppx, 2) - w2 = px_width / uppx + # w2 = px_width / uppx # scale up the width as the uppx get's large - w = uppx# * math.log(uppx, 666) + w = uppx # * math.log(uppx, 666) # ensure we make more then enough # frames (windows) for the output pixel @@ -269,18 +269,18 @@ def ds_m4( # we have room for all output down-samples. pts_per_pixel, r = divmod(xrange, frames) if r: - while r: - frames += 1 - pts_per_pixel, r = divmod(xrange, frames) - - print( - f'uppx: {uppx}\n' - f'xrange: {xrange}\n' - f'px_width: {px_width}\n' - f'pxw: {pxw}\n' - f'WTF w:{w}, w2:{w2}\n' - f'frames: {frames}\n' - ) + # while r: + frames += 1 + pts_per_pixel, r = divmod(xrange, frames) + + # print( + # f'uppx: {uppx}\n' + # f'xrange: {xrange}\n' + # f'px_width: {px_width}\n' + # f'pxw: {pxw}\n' + # f'WTF w:{w}, w2:{w2}\n' + # f'frames: {frames}\n' + # ) assert frames >= (xrange / uppx) # call into ``numba`` From 629ea8ba9d9c7dcbb0b6d955a7c6d75023484e39 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:09:30 -0400 Subject: [PATCH 024/113] Downsample on every uppx inrement since it's way faster --- piker/ui/_curve.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index e28035492..cf987203d 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -259,7 +259,7 @@ def downsample( y, px_width=px_width, uppx=uppx, - log_scale=bool(uppx) + # log_scale=bool(uppx) ) x = np.broadcast_to(x[:, None], y.shape) # x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() @@ -391,9 +391,9 @@ def update_from_array( if ( # std m4 downsample conditions px_width - and uppx_diff >= 4 - or uppx_diff <= -3 - or self._step_mode and abs(uppx_diff) >= 4 + and uppx_diff >= 1 + or uppx_diff <= -1 + or self._step_mode and abs(uppx_diff) >= 2 ): log.info( @@ -460,7 +460,7 @@ def update_from_array( self._in_ds = False - elif should_ds and px_width: + elif should_ds and px_width and uppx: x_out, y_out = self.downsample( x_out, y_out, From c5beecf8a1a1b3ffe5c24e68c57dc5d2ac4c1274 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 24 Apr 2022 17:09:58 -0400 Subject: [PATCH 025/113] Drop cursor debounce delay, decrease rate limit --- piker/ui/_cursor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index 43207b9f6..8f18fe458 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -43,8 +43,8 @@ # latency (in terms of perceived lag in cross hair) so really be sure # there's an improvement if you want to change it! -_mouse_rate_limit = 120 # TODO; should we calc current screen refresh rate? -_debounce_delay = 1 / 40 +_mouse_rate_limit = 60 # TODO; should we calc current screen refresh rate? +_debounce_delay = 0 _ch_label_opac = 1 From 12d60e6d9c0df35390fad59167717fb060099fd8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 08:34:53 -0400 Subject: [PATCH 026/113] WIP get incremental step curve updates working This took longer then i care to admit XD but it definitely adds a huge speedup and with only a few outstanding correctness bugs: - panning from left to right causes strange trailing artifacts in the flows fsp (vlm) sub-plot but only when some data is off-screen on the left but doesn't appear to be an issue if we keep the `._set_yrange()` handler hooked up to the `.sigXRangeChanged` signal (but we aren't going to because this makes panning way slower). i've got a feeling this is a bug todo with the device coordinate cache stuff and we may need to report to Qt core? - factoring out the step curve logic from `FastAppendCurve.update_from_array()` (un)fortunately required some logic branch uncoupling but also meant we needed special input controls to avoid things like redraws and curve appends for special cases, this will hopefully all be better rectified in code when the core of this method is moved into a renderer type/implementation. - the `tina_vwap` fsp curve now somehow causes hangs when doing erratic scrolling on downsampled graphics data. i have no idea why or how but disabling it makes the issue go away (ui will literally just freeze and gobble CPU on a `.paint()` call until you ctrl-c the hell out of it). my guess is that something in the logic for standard line curves and appends on large data sets is the issue? Code related changes/hacks: - drop use of `step_path_arrays_from_1d()`, it was always a bit hacky (being based on `pyqtgraph` internals) and was generally hard to understand since it returns 1d data instead of the more expected (N,2) array of "step levels"; instead this is now implemented (uglily) in the `Flow.update_graphics()` block for step curves (which will obviously get cleaned up and factored elsewhere). - add a bunch of new flags to the update method on the fast append curve: `draw_last: bool`, `slice_to_head: int`, `do_append: bool`, `should_redraw: bool` which are all controls to aid with previously mentioned issues specific to getting step curve updates working correctly. - add a ton of commented tinkering related code (that we may end up using) to both the flow and append curve methods that was written as part of the effort to get this all working. - implement all step curve updating inline in `Flow.update_graphics()` including prepend and append logic for pre-graphics incremental step data maintenance and in-view slicing as well as "last step" graphics updating. Obviously clean up commits coming stat B) --- piker/ui/_curve.py | 288 ++++++++++++++++++++++++++++----------------- piker/ui/_flows.py | 227 +++++++++++++++++++++++++++++------ 2 files changed, 373 insertions(+), 142 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index cf987203d..60353f081 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -45,74 +45,77 @@ # TODO: numba this instead.. -def step_path_arrays_from_1d( - x: np.ndarray, - y: np.ndarray, - include_endpoints: bool = False, +# def step_path_arrays_from_1d( +# x: np.ndarray, +# y: np.ndarray, +# include_endpoints: bool = True, + +# ) -> (np.ndarray, np.ndarray): +# ''' +# Generate a "step mode" curve aligned with OHLC style bars +# such that each segment spans each bar (aka "centered" style). + +# ''' +# # y_out = y.copy() +# # x_out = x.copy() + +# # x2 = np.empty( +# # # the data + 2 endpoints on either end for +# # # "termination of the path". +# # (len(x) + 1, 2), +# # # we want to align with OHLC or other sampling style +# # # bars likely so we need fractinal values +# # dtype=float, +# # ) + +# x2 = np.broadcast_to( +# x[:, None], +# ( +# x.size + 1, +# # 4, # only ohlc +# 2, +# ), +# ) + np.array([-0.5, 0.5]) + +# # x2[0] = x[0] - 0.5 +# # x2[1] = x[0] + 0.5 +# # x2[0, 0] = x[0] - 0.5 +# # x2[0, 1] = x[0] + 0.5 +# # x2[1:] = x[:, np.newaxis] + 0.5 +# # import pdbpp +# # pdbpp.set_trace() + +# # flatten to 1-d +# # x_out = x2.reshape(x2.size) +# # x_out = x2 + +# # we create a 1d with 2 extra indexes to +# # hold the start and (current) end value for the steps +# # on either end +# y2 = np.empty( +# (len(y) + 1, 2), +# dtype=y.dtype, +# ) +# y2[:] = y[:, np.newaxis] +# # y2[-1] = 0 -) -> (np.ndarray, np.ndarray): - ''' - Generate a "step mode" curve aligned with OHLC style bars - such that each segment spans each bar (aka "centered" style). +# # y_out = y2 - ''' - y_out = y.copy() - x_out = x.copy() - - # x2 = np.empty( - # # the data + 2 endpoints on either end for - # # "termination of the path". - # (len(x) + 1, 2), - # # we want to align with OHLC or other sampling style - # # bars likely so we need fractinal values - # dtype=float, - # ) - - x2 = np.broadcast_to( - x[:, None], - ( - x_out.size, - # 4, # only ohlc - 2, - ), - ) + np.array([-0.5, 0.5]) - - # x2[0] = x[0] - 0.5 - # x2[1] = x[0] + 0.5 - # x2[0, 0] = x[0] - 0.5 - # x2[0, 1] = x[0] + 0.5 - # x2[1:] = x[:, np.newaxis] + 0.5 - # import pdbpp - # pdbpp.set_trace() - - # flatten to 1-d - # x_out = x2.reshape(x2.size) - x_out = x2 - - # we create a 1d with 2 extra indexes to - # hold the start and (current) end value for the steps - # on either end - y2 = np.empty((len(y), 2), dtype=y.dtype) - y2[:] = y[:, np.newaxis] - y2[-1] = 0 - - y_out = y2 - -# y_out = np.empty( -# 2*len(y) + 2, -# dtype=y.dtype -# ) +# # y_out = np.empty( +# # 2*len(y) + 2, +# # dtype=y.dtype +# # ) - # flatten and set 0 endpoints - # y_out[1:-1] = y2.reshape(y2.size) - # y_out[0] = 0 - # y_out[-1] = 0 +# # flatten and set 0 endpoints +# # y_out[1:-1] = y2.reshape(y2.size) +# # y_out[0] = 0 +# # y_out[-1] = 0 - if not include_endpoints: - return x_out[:-1], y_out[:-1] +# if not include_endpoints: +# return x2[:-1], y2[:-1] - else: - return x_out, y_out +# else: +# return x2, y2 _line_styles: dict[str, int] = { @@ -158,6 +161,8 @@ def __init__( self._y = self.yData = y self._x = self.xData = x self._vr: Optional[tuple] = None + self._avr: Optional[tuple] = None + self._br = None self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -171,6 +176,7 @@ def __init__( # self._xrange: tuple[int, int] = self.dataBounds(ax=0) self._xrange: Optional[tuple[int, int]] = None + # self._x_iv_range = None # self._last_draw = time.time() self._in_ds: bool = False @@ -283,6 +289,10 @@ def update_from_array( view_range: Optional[tuple[int, int]] = None, profiler: Optional[pg.debug.Profiler] = None, + draw_last: bool = True, + slice_to_head: int = -1, + do_append: bool = True, + should_redraw: bool = False, ) -> QtGui.QPainterPath: ''' @@ -297,7 +307,7 @@ def update_from_array( disabled=not pg_profile_enabled(), gt=ms_slower_then, ) - # flip_cache = False + flip_cache = False if self._xrange: istart, istop = self._xrange @@ -330,7 +340,7 @@ def update_from_array( new_sample_rate = False should_ds = self._in_ds showing_src_data = self._in_ds - should_redraw = False + # should_redraw = False # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. @@ -342,32 +352,60 @@ def update_from_array( # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index - x_out, y_out = x_iv[:-1], y_iv[:-1] + x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] profiler(f'view range slice {view_range}') - ivl, ivr = view_range + vl, vr = view_range - probably_zoom_change = False + # last_ivr = self._x_iv_range + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + + zoom_or_append = False last_vr = self._vr + last_ivr = self._avr + if last_vr: - livl, livr = last_vr + # relative slice indices + lvl, lvr = last_vr + # abs slice indices + al, ar = last_ivr + + # append_length = int(x[-1] - istop) + # append_length = int(x_iv[-1] - ar) + + # left_change = abs(x_iv[0] - al) >= 1 + # right_change = abs(x_iv[-1] - ar) >= 1 + if ( - ivl < livl - or (ivr - livr) > 2 + # likely a zoom view change + (vr - lvr) > 2 or vl < lvl + # append / prepend update + # we had an append update where the view range + # didn't change but the data-viewed (shifted) + # underneath, so we need to redraw. + # or left_change and right_change and last_vr == view_range + + # not (left_change and right_change) and ivr + # ( + # or abs(x_iv[ivr] - livr) > 1 ): - probably_zoom_change = True + zoom_or_append = True + + # if last_ivr: + # liivl, liivr = last_ivr if ( view_range != last_vr and ( append_length > 1 - or probably_zoom_change + or zoom_or_append ) ): should_redraw = True # print("REDRAWING BRUH") self._vr = view_range + self._avr = x_iv[0], x_iv[slice_to_head] # x_last = x_iv[-1] # y_last = y_iv[-1] @@ -382,7 +420,7 @@ def update_from_array( # or self._in_ds # ): # by default we only pull data up to the last (current) index - x_out, y_out = x[:-1], y[:-1] + x_out, y_out = x[:slice_to_head], y[:slice_to_head] if prepend_length > 0: should_redraw = True @@ -434,12 +472,12 @@ def update_from_array( # step mode: draw flat top discrete "step" # over the index space for each datum. # if self._step_mode: + # self.disable_cache() + # flip_cache = True # x_out, y_out = step_path_arrays_from_1d( # x_out, # y_out, # ) - # # self.disable_cache() - # # flip_cache = True # # TODO: numba this bish # profiler('generated step arrays') @@ -460,7 +498,7 @@ def update_from_array( self._in_ds = False - elif should_ds and px_width and uppx: + elif should_ds and uppx and px_width > 1: x_out, y_out = self.downsample( x_out, y_out, @@ -477,11 +515,9 @@ def update_from_array( finiteCheck=False, path=self.path, ) + self.prepareGeometryChange() profiler( - 'generated fresh path\n' - f'should_redraw: {should_redraw}\n' - f'should_ds: {should_ds}\n' - f'new_sample_rate: {new_sample_rate}\n' + f'generated fresh path. (should_redraw: {should_redraw} should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') @@ -514,26 +550,29 @@ def update_from_array( elif ( append_length > 0 + and do_append + and not should_redraw # and not view_range ): - new_x = x[-append_length - 2:-1] - new_y = y[-append_length - 2:-1] + print(f'{self._name} append len: {append_length}') + new_x = x[-append_length - 2:slice_to_head] + new_y = y[-append_length - 2:slice_to_head] profiler('sliced append path') # if self._step_mode: - # new_x, new_y = step_path_arrays_from_1d( - # new_x, - # new_y, - # ) - # # [1:] since we don't need the vertical line normally at - # # the beginning of the step curve taking the first (x, - # # y) poing down to the x-axis **because** this is an - # # appended path graphic. - # new_x = new_x[1:] - # new_y = new_y[1:] - - # # self.disable_cache() - # # flip_cache = True + # # new_x, new_y = step_path_arrays_from_1d( + # # new_x, + # # new_y, + # # ) + # # # [1:] since we don't need the vertical line normally at + # # # the beginning of the step curve taking the first (x, + # # # y) poing down to the x-axis **because** this is an + # # # appended path graphic. + # # new_x = new_x[1:] + # # new_y = new_y[1:] + + # self.disable_cache() + # flip_cache = True # profiler('generated step data') @@ -563,7 +602,7 @@ def update_from_array( # an attempt at trying to make append-updates faster.. if self.fast_path is None: self.fast_path = append_path - self.fast_path.reserve(int(6e3)) + # self.fast_path.reserve(int(6e3)) else: self.fast_path.connectPath(append_path) size = self.fast_path.capacity() @@ -596,19 +635,20 @@ def update_from_array( # self.disable_cache() # flip_cache = True - self.draw_last(x, y) - profiler('draw last segment') + if draw_last: + self.draw_last(x, y) + profiler('draw last segment') + + + # if flip_cache: + # # # XXX: seems to be needed to avoid artifacts (see above). + # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) # trigger redraw of path # do update before reverting to cache mode - # self.prepareGeometryChange() self.update() profiler('.update()') - # if flip_cache: - # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - def draw_last( self, x: np.ndarray, @@ -624,10 +664,14 @@ def draw_last( self._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, + # x_last, 0, + # x_last, 0, ) self._last_step_rect = QRectF( x_last - 0.5, 0, x_last + 0.5, y_last + # x_last, 0, + # x_last, y_last ) # print( # f"path br: {self.path.boundingRect()}", @@ -640,6 +684,8 @@ def draw_last( x_last, y_last ) + self.update() + # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): @@ -685,7 +731,7 @@ def disable_cache(self) -> None: # XXX: pretty annoying but, without this there's little # artefacts on the append updates to the curve... self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - self.prepareGeometryChange() + # self.prepareGeometryChange() def boundingRect(self): ''' @@ -705,6 +751,7 @@ def _path_br(self): ''' hb = self.path.controlPointRect() + # hb = self.path.boundingRect() hb_size = hb.size() fp = self.fast_path @@ -713,17 +760,47 @@ def _path_br(self): hb_size = fhb.size() + hb_size # print(f'hb_size: {hb_size}') + # if self._last_step_rect: + # hb_size += self._last_step_rect.size() + + # if self._line: + # br = self._last_step_rect.bottomRight() + + # tl = QPointF( + # # self._vr[0], + # # hb.topLeft().y(), + # # 0, + # # hb_size.height() + 1 + # ) + + # if self._last_step_rect: + # br = self._last_step_rect.bottomRight() + + # else: + # hb_size += QSizeF(1, 1) w = hb_size.width() + 1 h = hb_size.height() + 1 + # br = QPointF( + # self._vr[-1], + # # tl.x() + w, + # tl.y() + h, + # ) + br = QRectF( # top left + # hb.topLeft() + # tl, QPointF(hb.topLeft()), + # br, # total size + # QSizeF(hb_size) + # hb_size, QSizeF(w, h) ) + self._br = br # print(f'bounding rect: {br}') return br @@ -740,6 +817,7 @@ def paint( disabled=not pg_profile_enabled(), gt=ms_slower_then, ) + self.prepareGeometryChange() if ( self._step_mode diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index b150a2d1b..03d95a356 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,6 +34,13 @@ from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath +from PyQt5.QtCore import ( + # Qt, + QLineF, + # QSizeF, + QRectF, + # QPointF, +) from ..data._sharedmem import ( ShmArray, @@ -465,7 +472,7 @@ def update_graphics( self.gy[ ishm_first:iflat_first ] = rfn.structured_to_unstructured( - self.shm.array[fields][:iflat_first] + self.shm._array[fields][ishm_first:iflat_first] ) self._iflat_first = ishm_first @@ -516,6 +523,8 @@ def update_graphics( y_iv=y_iv, view_range=(ivl, ivr), # hack profiler=profiler, + # should_redraw=False, + # do_append=False, ) curve.show() profiler('updated ds curve') @@ -578,21 +587,36 @@ def update_graphics( # graphics.draw_last(last) else: - + # ``FastAppendCurve`` case: array_key = array_key or self.name - # ``FastAppendCurve`` case: if graphics._step_mode and self.gy is None: + self._iflat_first = self.shm._first.value # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm # fields = ['index', array_key] - i = shm._array['index'] - out = shm._array[array_key] + i = shm._array['index'].copy() + out = shm._array[array_key].copy() + + self.gx = np.broadcast_to( + i[:, None], + (i.size, 2), + ) + np.array([-0.5, 0.5]) + + + # self.gy = np.broadcast_to( + # out[:, None], (out.size, 2), + # ) + self.gy = np.empty((len(out), 2), dtype=out.dtype) + self.gy[:] = out[:, np.newaxis] - self.gx, self.gy = step_path_arrays_from_1d(i, out) + # start y at origin level + self.gy[0, 0] = 0 + + # self.gx, self.gy = step_path_arrays_from_1d(i, out) # flat = self.gy = self.shm.unstruct_view(fields) # self.gy = self.shm.ustruct(fields) @@ -635,17 +659,29 @@ def update_graphics( self.shm._first.value ) + il = max(iflat - 1, 0) + # check for shm prepend updates since last read. if iflat_first != ishm_first: - # write newly prepended data to flattened copy - _gx, self.gy[ - ishm_first:iflat_first - ] = step_path_arrays_from_1d( - self.shm.array['index'][:iflat_first], - self.shm.array[array_key][:iflat_first], + print(f'prepend {array_key}') + + i_prepend = self.shm._array['index'][ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ishm_first:iflat_first] + + y2_prepend = np.broadcast_to( + y_prepend[:, None], (y_prepend.size, 2), ) + + # write newly prepended data to flattened copy + self.gy[ishm_first:iflat_first] = y2_prepend + # ] = step_path_arrays_from_1d( + # ] = step_path_arrays_from_1d( + # i_prepend, + # y_prepend, + # ) self._iflat_first = ishm_first + # # flat = self.gy = self.shm.unstruct_view(fields) # self.gy = self.shm.ustruct(fields) # # self._iflat_last = self.shm._last.value @@ -654,40 +690,112 @@ def update_graphics( # # do an update for the most recent prepend # # index # iflat = ishm_first - if iflat != ishm_last: - _x, to_update = step_path_arrays_from_1d( - self.shm._array[iflat:ishm_last]['index'], - self.shm._array[iflat:ishm_last][array_key], + append_diff = ishm_last - iflat + # if iflat != ishm_last: + if append_diff: + + # slice up to the last datum since last index/append update + new_x = self.shm._array[il:ishm_last]['index']#.copy() + new_y = self.shm._array[il:ishm_last][array_key]#.copy() + + # _x, to_update = step_path_arrays_from_1d(new_x, new_y) + + # new_x2 = = np.broadcast_to( + # new_x2[:, None], + # (new_x2.size, 2), + # ) + np.array([-0.5, 0.5]) + + new_y2 = np.broadcast_to( + new_y[:, None], (new_y.size, 2), ) + # new_y2 = np.empty((len(new_y), 2), dtype=new_y.dtype) + # new_y2[:] = new_y[:, np.newaxis] + + # import pdbpp + # pdbpp.set_trace() + + # print( + # f'updating step curve {to_update}\n' + # f'last array val: {new_x}, {new_y}' + # ) # to_update = rfn.structured_to_unstructured( # self.shm._array[iflat:ishm_last][fields] # ) - # import pdbpp - # pdbpp.set_trace() - self.gy[iflat:ishm_last-1] = to_update - self.gy[-1] = 0 - print(f'updating step curve {to_update}') + # if not to_update.any(): + # if new_y.any() and not to_update.any(): + # import pdbpp + # pdbpp.set_trace() + + # print(f'{array_key} new values new_x:{new_x}, new_y:{new_y}') + # head, last = to_update[:-1], to_update[-1] + self.gy[il:ishm_last] = new_y2 + + gy = self.gy[il:ishm_last] + + # self.gy[-1] = to_update[-1] profiler('updated step curve data') + # print( + # f'append size: {append_diff}\n' + # f'new_x: {new_x}\n' + # f'new_y: {new_y}\n' + # f'new_y2: {new_y2}\n' + # f'new gy: {gy}\n' + # ) + + # update local last-index tracking + self._iflat_last = ishm_last + + # ( + # iflat_first, + # iflat, + # ishm_last, + # ishm_first, + # ) = ( + # self._iflat_first, + # self._iflat_last, + # self.shm._last.value, + # self.shm._first.value + # ) + # graphics.draw_last(last['index'], last[array_key]) + # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last] + x_step = self.gx[ishm_first:ishm_last+2] + # x_step[-1] = last['index'] + # x_step[-1] = last['index'] + # to 1d x = x_step.reshape(-1) - y_step = self.gy[ishm_first:ishm_last] + + y_step = self.gy[ishm_first:ishm_last+2] + lasts = self.shm.array[['index', array_key]] + last = lasts[array_key][-1] + y_step[-1] = last + # to 1d y = y_step.reshape(-1) - profiler('sliced step data') + # y[-1] = 0 - # update local last-index tracking - self._iflat_last = ishm_last + # s = 6 + # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') + + profiler('sliced step data') # reshape to 1d for graphics rendering # y = y_flat.reshape(-1) # x = x_flat.reshape(-1) # do all the same for only in-view data - y_iv = y_step[ivl:ivr].reshape(-1) - x_iv = x_step[ivl:ivr].reshape(-1) + ys_iv = y_step[ivl:ivr+1] + xs_iv = x_step[ivl:ivr+1] + y_iv = ys_iv.reshape(ys_iv.size) + x_iv = xs_iv.reshape(xs_iv.size) + # print( + # f'ys_iv : {ys_iv[-s:]}\n' + # f'y_iv: {y_iv[-s:]}\n' + # f'xs_iv: {xs_iv[-s:]}\n' + # f'x_iv: {x_iv[-s:]}\n' + # ) # y_iv = y_iv_flat.reshape(-1) # x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') @@ -696,7 +804,49 @@ def update_graphics( # x, y = ohlc_flatten(array) # x_iv, y_iv = ohlc_flatten(in_view) # profiler('flattened OHLC data') - graphics.reset_cache() + + x_last = array['index'][-1] + y_last = array[array_key][-1] + graphics._last_line = QLineF( + x_last - 0.5, 0, + x_last + 0.5, 0, + # x_last, 0, + # x_last, 0, + ) + graphics._last_step_rect = QRectF( + x_last - 0.5, 0, + x_last + 0.5, y_last, + # x_last, 0, + # x_last, y_last + ) + # graphics.update() + + graphics.update_from_array( + x=x, + y=y, + + x_iv=x_iv, + y_iv=y_iv, + + view_range=(ivl, ivr) if use_vr else None, + + draw_last=False, + slice_to_head=-2, + + should_redraw=bool(append_diff), + # do_append=False, + + **kwargs + ) + # graphics.reset_cache() + # print( + # f"path br: {graphics.path.boundingRect()}\n", + # # f"fast path br: {graphics.fast_path.boundingRect()}", + # f"last rect br: {graphics._last_step_rect}\n", + # f"full br: {graphics._br}\n", + # ) + + # graphics.boundingRect() else: x = array['index'] @@ -704,17 +854,20 @@ def update_graphics( x_iv = in_view['index'] y_iv = in_view[array_key] - graphics.update_from_array( - x=x, - y=y, + # graphics.draw_last(x, y) + profiler('draw last segment {array_key}') - x_iv=x_iv, - y_iv=y_iv, + graphics.update_from_array( + x=x, + y=y, - view_range=(ivl, ivr) if use_vr else None, + x_iv=x_iv, + y_iv=y_iv, + + view_range=(ivl, ivr) if use_vr else None, - **kwargs - ) + **kwargs + ) return graphics From 186658ab090b646e05ba91ebb051f5e5850a75da Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 08:52:34 -0400 Subject: [PATCH 027/113] Drop uppx guard around downsamples on interaction Since downsampling with the more correct version of m4 (uppx driven windows sizing) is super fast now we don't need to avoid downsampling on low uppx values. Further all graphics objects now support in-view slicing so make sure to use it on interaction updates. Pass in the view profiler to update method calls for more detailed measuring. Even moar, - Add a manual call to `.maybe_downsample_graphics()` inside the mouse wheel event handler since it seems that sometimes trailing events get lost from the `.sigRangeChangedManually` signal which can result in "non-downsampled-enough" graphics on chart given the scroll amount; this manual call seems to entirely fix this? - drop "max zoom" guard since internals now support (near) infinite scroll out to graphics becoming a single pixel column line XD - add back in commented xrange signal connect code for easy testing to verify against range updates not happening without it --- piker/ui/_interaction.py | 71 +++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 943f33709..a2c99e38d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -34,10 +34,9 @@ from ..log import get_logger from .._profile import pg_profile_enabled, ms_slower_then -from ._style import _min_points_to_show +# from ._style import _min_points_to_show from ._editors import SelectRect from . import _event -from ._ohlc import BarItems log = get_logger(__name__) @@ -485,11 +484,11 @@ def wheelEvent( # don't zoom more then the min points setting l, lbar, rbar, r = chart.bars_range() - vl = r - l + # vl = r - l - if ev.delta() > 0 and vl <= _min_points_to_show: - log.debug("Max zoom bruh...") - return + # if ev.delta() > 0 and vl <= _min_points_to_show: + # log.debug("Max zoom bruh...") + # return # if ( # ev.delta() < 0 @@ -570,6 +569,17 @@ def wheelEvent( self._resetTarget() self.scaleBy(s, focal) + + # XXX: without this is seems as though sometimes + # when zooming in from far out (and maybe vice versa?) + # the signal isn't being fired enough since if you pan + # just after you'll see further downsampling code run + # (pretty noticeable on the OHLC ds curve) but with this + # that never seems to happen? Only question is how much this + # "double work" is causing latency when these missing event + # fires don't happen? + self.maybe_downsample_graphics() + self.sigRangeChangedManually.emit(mask) # self._ic.set() @@ -736,7 +746,7 @@ def _set_yrange( # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = True, + autoscale_linked_plots: bool = False, name: Optional[str] = None, # autoscale_overlays: bool = False, @@ -804,7 +814,7 @@ def _set_yrange( # for chart in plots: # if chart and not chart._static_yrange: # chart.cv._set_yrange( - # bars_range=br, + # # bars_range=br, # autoscale_linked_plots=False, # ) # profiler('autoscaled linked plots') @@ -858,9 +868,6 @@ def enable_auto_yrange( # splitter(s) resizing src_vb.sigResized.connect(self._set_yrange) - # mouse wheel doesn't emit XRangeChanged - src_vb.sigRangeChangedManually.connect(self._set_yrange) - # TODO: a smarter way to avoid calling this needlessly? # 2 things i can think of: # - register downsample-able graphics specially and only @@ -871,9 +878,15 @@ def enable_auto_yrange( self.maybe_downsample_graphics ) - def disable_auto_yrange( - self, - ) -> None: + # mouse wheel doesn't emit XRangeChanged + src_vb.sigRangeChangedManually.connect(self._set_yrange) + + # src_vb.sigXRangeChanged.connect(self._set_yrange) + # src_vb.sigXRangeChanged.connect( + # self.maybe_downsample_graphics + # ) + + def disable_auto_yrange(self) -> None: self.sigResized.disconnect( self._set_yrange, @@ -885,6 +898,11 @@ def disable_auto_yrange( self._set_yrange, ) + # self.sigXRangeChanged.disconnect(self._set_yrange) + # self.sigXRangeChanged.disconnect( + # self.maybe_downsample_graphics + # ) + def x_uppx(self) -> float: ''' Return the "number of x units" within a single @@ -905,13 +923,6 @@ def x_uppx(self) -> float: def maybe_downsample_graphics(self): - uppx = self.x_uppx() - # if not ( - # # we probably want to drop this once we are "drawing in - # # view" for downsampled flows.. - # uppx and uppx > 6 - # and self._ic is not None - # ): profiler = pg.debug.Profiler( msg=f'ChartView.maybe_downsample_graphics() for {self.name}', disabled=not pg_profile_enabled(), @@ -922,7 +933,7 @@ def maybe_downsample_graphics(self): # the profiler in the delegated method calls. delayed=False, # gt=3, - # gt=ms_slower_then, + gt=ms_slower_then, ) # TODO: a faster single-loop-iterator way of doing this XD @@ -940,12 +951,6 @@ def maybe_downsample_graphics(self): ): continue - graphics = flow.graphics - - # use_vr = False - # if isinstance(graphics, BarItems): - # use_vr = True - # pass in no array which will read and render from the last # passed array (normally provided by the display loop.) chart.update_graphics_from_flow( @@ -953,17 +958,9 @@ def maybe_downsample_graphics(self): use_vr=True, # gets passed down into graphics obj - # profiler=profiler, + profiler=profiler, ) profiler(f'range change updated {chart_name}:{name}') profiler.finish() - # else: - # # don't bother updating since we're zoomed out bigly and - # # in a pan-interaction, in which case we shouldn't be - # # doing view-range based rendering (at least not yet). - # # print(f'{uppx} exiting early!') - # profiler(f'dowsampling skipped - not in uppx range {uppx} <= 16') - - # profiler.finish() From b12921678b286e0d30ef261e721d11be6d04b134 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:04 -0400 Subject: [PATCH 028/113] Drop step routine import --- piker/ui/_flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 03d95a356..d2b1fa90d 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -53,7 +53,7 @@ ) from ._curve import ( FastAppendCurve, - step_path_arrays_from_1d, + # step_path_arrays_from_1d, ) from ._compression import ( # ohlc_flatten, From 859eaffa2950873559ff39e3bc06ec5b87231280 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:24 -0400 Subject: [PATCH 029/113] Drop vwap fsp for now; causes hangs.. --- piker/ui/_fsp.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 9aa10fb3c..5ed85d9b6 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -75,6 +75,7 @@ def update_fsp_chart( flow, graphics_name: str, array_key: Optional[str], + **kwargs, ) -> None: @@ -96,6 +97,7 @@ def update_fsp_chart( chart.update_graphics_from_flow( graphics_name, array_key=array_key or graphics_name, + **kwargs, ) # XXX: re: ``array_key``: fsp func names must be unique meaning we @@ -884,10 +886,10 @@ def chart_curves( # built-in vlm fsps for target, conf in { - tina_vwap: { - 'overlay': 'ohlc', # overlays with OHLCV (main) chart - 'anchor': 'session', - }, + # tina_vwap: { + # 'overlay': 'ohlc', # overlays with OHLCV (main) chart + # 'anchor': 'session', + # }, }.items(): started = await admin.open_fsp_chart( target, From 2b6041465ccf05c36c408a4ecbc8db7c49678431 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:27:38 -0400 Subject: [PATCH 030/113] Startup up with 3k bars --- piker/ui/_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index cecbbff5c..8a95327fc 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -856,7 +856,7 @@ def marker_right_points( def default_view( self, - bars_from_y: int = 5000, + bars_from_y: int = 3000, ) -> None: ''' From 0770a39125eab777b206619636716074f6fb2aa2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 26 Apr 2022 09:28:09 -0400 Subject: [PATCH 031/113] Only do curve appends on low uppx levels --- piker/ui/_display.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index b82d1253f..6c8bdddff 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -331,7 +331,7 @@ def graphics_update_cycle( vars = ds.vars tick_margin = vars['tick_margin'] - update_uppx = 6 + update_uppx = 16 for sym, quote in ds.quotes.items(): @@ -392,7 +392,8 @@ def graphics_update_cycle( if ( ( - xpx < update_uppx or i_diff > 0 + xpx < update_uppx + or i_diff > 0 and liv ) or trigger_all @@ -401,7 +402,6 @@ def graphics_update_cycle( # once the $vlm is up? vlm_chart.update_graphics_from_flow( 'volume', - # UGGGh, see ``maxmin()`` impl in `._fsp` for # the overlayed plotitems... we need a better # bay to invoke a maxmin per overlay.. @@ -435,6 +435,7 @@ def graphics_update_cycle( flow, curve_name, array_key=curve_name, + do_append=xpx < update_uppx, ) # is this even doing anything? # (pretty sure it's the real-time @@ -496,6 +497,7 @@ def graphics_update_cycle( ): chart.update_graphics_from_flow( chart.name, + do_append=xpx < update_uppx, ) # iterate in FIFO order per tick-frame From 0744dd041517054d72ce9ad35da1d6fedc982091 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 27 Apr 2022 17:18:11 -0400 Subject: [PATCH 032/113] Up the display throttle rate to 22Hz --- piker/ui/_display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 6c8bdddff..bbc708fd3 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -63,7 +63,7 @@ log = get_logger(__name__) # TODO: load this from a config.toml! -_quote_throttle_rate: int = 12 # Hz +_quote_throttle_rate: int = 22 # Hz # a working tick-type-classes template From 7a3437348ddb319322fef20913c1090d1567ac2d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 27 Apr 2022 17:19:08 -0400 Subject: [PATCH 033/113] An absolute uppx diff of >= 1 seems more then fine --- piker/ui/_curve.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 60353f081..66862086c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -429,10 +429,7 @@ def update_from_array( if ( # std m4 downsample conditions px_width - and uppx_diff >= 1 - or uppx_diff <= -1 - or self._step_mode and abs(uppx_diff) >= 2 - + and abs(uppx_diff) >= 1 ): log.info( f'{self._name} sampler change: {self._last_uppx} -> {uppx}' From 36a10155bc7e3cc0e5a9416fb0064db0e2d23448 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 29 Apr 2022 11:24:21 -0400 Subject: [PATCH 034/113] Add profiler passthrough type annot, comments about appends vs. uppx --- piker/ui/_flows.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d2b1fa90d..4901a6e70 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -320,7 +320,7 @@ def update_graphics( render: bool = True, array_key: Optional[str] = None, - profiler=None, + profiler: Optional[pg.debug.Profiler] = None, **kwargs, @@ -524,7 +524,10 @@ def update_graphics( view_range=(ivl, ivr), # hack profiler=profiler, # should_redraw=False, - # do_append=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, + **kwargs, ) curve.show() profiler('updated ds curve') @@ -589,6 +592,7 @@ def update_graphics( else: # ``FastAppendCurve`` case: array_key = array_key or self.name + uppx = graphics.x_uppx() if graphics._step_mode and self.gy is None: self._iflat_first = self.shm._first.value @@ -834,7 +838,9 @@ def update_graphics( slice_to_head=-2, should_redraw=bool(append_diff), - # do_append=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, **kwargs ) @@ -866,6 +872,8 @@ def update_graphics( view_range=(ivl, ivr) if use_vr else None, + # NOTE: already passed through by display loop? + # do_append=uppx < 16, **kwargs ) From e163a7e3369b5dbcc905a459bbcc40bb7c9fe450 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 29 Apr 2022 11:27:18 -0400 Subject: [PATCH 035/113] Drop `bar_wap` curve for now, seems to also be causing hangs?! --- piker/ui/_display.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index bbc708fd3..29b4c6d42 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -697,15 +697,17 @@ async def display_symbol_data( # plot historical vwap if available wap_in_history = False - if brokermod._show_wap_in_history: - - if 'bar_wap' in bars.dtype.fields: - wap_in_history = True - chart.draw_curve( - name='bar_wap', - data=bars, - add_label=False, - ) + # XXX: FOR SOME REASON THIS IS CAUSING HANGZ!?! + # if brokermod._show_wap_in_history: + + # if 'bar_wap' in bars.dtype.fields: + # wap_in_history = True + # chart.draw_curve( + # name='bar_wap', + # shm=ohlcv, + # color='default_light', + # add_label=False, + # ) # size view to data once at outset chart.cv._set_yrange() From fb38265199e609f42925263de24624fe64f87073 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 1 May 2022 19:13:21 -0400 Subject: [PATCH 036/113] Clean out legacy code from `Flow.update_graphics()` --- piker/ui/_flows.py | 125 ++++----------------------------------------- 1 file changed, 10 insertions(+), 115 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 4901a6e70..707be6836 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -377,8 +377,6 @@ def update_graphics( ) curve = FastAppendCurve( - # y=y, - # x=x, name='OHLC', color=graphics._color, ) @@ -610,7 +608,6 @@ def update_graphics( (i.size, 2), ) + np.array([-0.5, 0.5]) - # self.gy = np.broadcast_to( # out[:, None], (out.size, 2), # ) @@ -620,36 +617,6 @@ def update_graphics( # start y at origin level self.gy[0, 0] = 0 - # self.gx, self.gy = step_path_arrays_from_1d(i, out) - - # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # first = self._iflat_first = self.shm._first.value - # last = self._iflat_last = self.shm._last.value - - # # write pushed data to flattened copy - # self.gy[first:last] = rfn.structured_to_unstructured( - # self.shm.array[fields] - # ) - - # # generate an flat-interpolated x-domain - # self.gx = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # ( - # shm._array.size, - # # 4, # only ohlc - # self.gy.shape[1], - # ), - # ) + np.array([-0.5, 0, 0, 0.5]) - # ) - # assert self.gy.any() - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.gy._first.value - # update flatted ohlc copy - if graphics._step_mode: ( iflat_first, @@ -670,8 +637,11 @@ def update_graphics( print(f'prepend {array_key}') - i_prepend = self.shm._array['index'][ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ishm_first:iflat_first] + # i_prepend = self.shm._array['index'][ + # ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ + ishm_first:iflat_first + ] y2_prepend = np.broadcast_to( y_prepend[:, None], (y_prepend.size, 2), @@ -679,66 +649,19 @@ def update_graphics( # write newly prepended data to flattened copy self.gy[ishm_first:iflat_first] = y2_prepend - # ] = step_path_arrays_from_1d( - # ] = step_path_arrays_from_1d( - # i_prepend, - # y_prepend, - # ) self._iflat_first = ishm_first - # # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # # self._iflat_last = self.shm._last.value - - # # self._iflat_first = self.shm._first.value - # # do an update for the most recent prepend - # # index - # iflat = ishm_first append_diff = ishm_last - iflat - # if iflat != ishm_last: if append_diff: # slice up to the last datum since last index/append update - new_x = self.shm._array[il:ishm_last]['index']#.copy() - new_y = self.shm._array[il:ishm_last][array_key]#.copy() - - # _x, to_update = step_path_arrays_from_1d(new_x, new_y) - - # new_x2 = = np.broadcast_to( - # new_x2[:, None], - # (new_x2.size, 2), - # ) + np.array([-0.5, 0.5]) + # new_x = self.shm._array[il:ishm_last]['index'] + new_y = self.shm._array[il:ishm_last][array_key] new_y2 = np.broadcast_to( new_y[:, None], (new_y.size, 2), ) - # new_y2 = np.empty((len(new_y), 2), dtype=new_y.dtype) - # new_y2[:] = new_y[:, np.newaxis] - - # import pdbpp - # pdbpp.set_trace() - - # print( - # f'updating step curve {to_update}\n' - # f'last array val: {new_x}, {new_y}' - # ) - - # to_update = rfn.structured_to_unstructured( - # self.shm._array[iflat:ishm_last][fields] - # ) - - # if not to_update.any(): - # if new_y.any() and not to_update.any(): - # import pdbpp - # pdbpp.set_trace() - - # print(f'{array_key} new values new_x:{new_x}, new_y:{new_y}') - # head, last = to_update[:-1], to_update[-1] self.gy[il:ishm_last] = new_y2 - - gy = self.gy[il:ishm_last] - - # self.gy[-1] = to_update[-1] profiler('updated step curve data') # print( @@ -752,43 +675,23 @@ def update_graphics( # update local last-index tracking self._iflat_last = ishm_last - # ( - # iflat_first, - # iflat, - # ishm_last, - # ishm_first, - # ) = ( - # self._iflat_first, - # self._iflat_last, - # self.shm._last.value, - # self.shm._first.value - # ) - # graphics.draw_last(last['index'], last[array_key]) - # slice out up-to-last step contents x_step = self.gx[ishm_first:ishm_last+2] - # x_step[-1] = last['index'] - # x_step[-1] = last['index'] - # to 1d + # shape to 1d x = x_step.reshape(-1) y_step = self.gy[ishm_first:ishm_last+2] lasts = self.shm.array[['index', array_key]] last = lasts[array_key][-1] y_step[-1] = last - # to 1d + # shape to 1d y = y_step.reshape(-1) - # y[-1] = 0 # s = 6 # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') profiler('sliced step data') - # reshape to 1d for graphics rendering - # y = y_flat.reshape(-1) - # x = x_flat.reshape(-1) - # do all the same for only in-view data ys_iv = y_step[ivl:ivr+1] xs_iv = x_step[ivl:ivr+1] @@ -800,8 +703,6 @@ def update_graphics( # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - # y_iv = y_iv_flat.reshape(-1) - # x_iv = x_iv_flat.reshape(-1) profiler('flattened ustruct in-view OHLC data') # legacy full-recompute-everytime method @@ -814,14 +715,10 @@ def update_graphics( graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, - # x_last, 0, - # x_last, 0, ) graphics._last_step_rect = QRectF( x_last - 0.5, 0, x_last + 0.5, y_last, - # x_last, 0, - # x_last, y_last ) # graphics.update() @@ -852,8 +749,6 @@ def update_graphics( # f"full br: {graphics._br}\n", # ) - # graphics.boundingRect() - else: x = array['index'] y = array[array_key] @@ -861,7 +756,7 @@ def update_graphics( y_iv = in_view[array_key] # graphics.draw_last(x, y) - profiler('draw last segment {array_key}') + profiler(f'draw last segment {array_key}') graphics.update_from_array( x=x, From 1fcb9233b452ecde3e58be5928c384075425e778 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 May 2022 11:40:53 -0400 Subject: [PATCH 037/113] Add back mx/mn updates for L1-in-view, lost during rebase --- piker/ui/_display.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index 29b4c6d42..aa7761dba 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -366,7 +366,7 @@ def graphics_update_cycle( mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin profiler('maxmin call') - liv = r > i_step # the last datum is in view + liv = r >= i_step # the last datum is in view # don't real-time "shift" the curve to the # left unless we get one of the following: @@ -374,7 +374,6 @@ def graphics_update_cycle( ( i_diff > 0 # no new sample step and xpx < 4 # chart is zoomed out very far - and r >= i_step # the last datum isn't in view and liv ) or trigger_all @@ -589,6 +588,7 @@ def graphics_update_cycle( main_vb._ic is None or not main_vb._ic.is_set() ): + # print(f'updating range due to mxmn') main_vb._set_yrange( # TODO: we should probably scale # the view margin based on the size @@ -599,7 +599,8 @@ def graphics_update_cycle( yrange=(mn, mx), ) - vars['last_mx'], vars['last_mn'] = mx, mn + # XXX: update this every draw cycle to make L1-always-in-view work. + vars['last_mx'], vars['last_mn'] = mx, mn # run synchronous update on all linked flows for curve_name, flow in chart._flows.items(): From 4f36743f6415dd4b232764f0ae393ca799b8802b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 9 May 2022 10:26:44 -0400 Subject: [PATCH 038/113] Only udpate prepended graphics when actually in view --- piker/data/_sampling.py | 19 +++++++++++++++---- piker/data/feed.py | 9 +++++++++ piker/fsp/_engine.py | 7 ++++++- piker/ui/_display.py | 14 +++++++++++++- piker/ui/_fsp.py | 8 ++++++-- 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 466ef0e71..10dc43f63 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -142,11 +142,17 @@ async def broadcast( shm: Optional[ShmArray] = None, ) -> None: - # broadcast the buffer index step to any subscribers for - # a given sample period. + ''' + Broadcast the given ``shm: ShmArray``'s buffer index step to any + subscribers for a given sample period. + + The sent msg will include the first and last index which slice into + the buffer's non-empty data. + + ''' subs = sampler.subscribers.get(delay_s, ()) - last = -1 + first = last = -1 if shm is None: periods = sampler.ohlcv_shms.keys() @@ -156,11 +162,16 @@ async def broadcast( if periods: lowest = min(periods) shm = sampler.ohlcv_shms[lowest][0] + first = shm._first.value last = shm._last.value for stream in subs: try: - await stream.send({'index': last}) + await stream.send({ + 'first': first, + 'last': last, + 'index': last, + }) except ( trio.BrokenResourceError, trio.ClosedResourceError diff --git a/piker/data/feed.py b/piker/data/feed.py index 605349e9c..848fcc108 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -795,6 +795,15 @@ async def manage_history( # manually trigger step update to update charts/fsps # which need an incremental update. + # NOTE: the way this works is super duper + # un-intuitive right now: + # - the broadcaster fires a msg to the fsp subsystem. + # - fsp subsys then checks for a sample step diff and + # possibly recomputes prepended history. + # - the fsp then sends back to the parent actor + # (usually a chart showing graphics for said fsp) + # which tells the chart to conduct a manual full + # graphics loop cycle. for delay_s in sampler.subscribers: await broadcast(delay_s) diff --git a/piker/fsp/_engine.py b/piker/fsp/_engine.py index 0776c7a24..cf45c40e3 100644 --- a/piker/fsp/_engine.py +++ b/piker/fsp/_engine.py @@ -369,7 +369,12 @@ async def resync( # always trigger UI refresh after history update, # see ``piker.ui._fsp.FspAdmin.open_chain()`` and # ``piker.ui._display.trigger_update()``. - await client_stream.send('update') + await client_stream.send({ + 'fsp_update': { + 'key': dst_shm_token, + 'first': dst._first.value, + 'last': dst._last.value, + }}) return tracker, index def is_synced( diff --git a/piker/ui/_display.py b/piker/ui/_display.py index aa7761dba..d0654b10d 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -309,6 +309,7 @@ def graphics_update_cycle( ds: DisplayState, wap_in_history: bool = False, trigger_all: bool = False, # flag used by prepend history updates + prepend_update_index: Optional[int] = None, ) -> None: # TODO: eventually optimize this whole graphics stack with ``numba`` @@ -368,6 +369,17 @@ def graphics_update_cycle( profiler('maxmin call') liv = r >= i_step # the last datum is in view + if ( + prepend_update_index is not None + and lbar > prepend_update_index + ): + # on a history update (usually from the FSP subsys) + # if the segment of history that is being prepended + # isn't in view there is no reason to do a graphics + # update. + log.debug('Skipping prepend graphics cycle: frame not in view') + return + # don't real-time "shift" the curve to the # left unless we get one of the following: if ( @@ -639,7 +651,7 @@ async def display_symbol_data( ) # historical data fetch - brokermod = brokers.get_brokermod(provider) + # brokermod = brokers.get_brokermod(provider) # ohlc_status_done = sbar.open_status( # 'retreiving OHLC history.. ', diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 5ed85d9b6..3d90f0140 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -435,12 +435,16 @@ async def open_chain( # wait for graceful shutdown signal async with stream.subscribe() as stream: async for msg in stream: - if msg == 'update': + info = msg.get('fsp_update') + if info: # if the chart isn't hidden try to update # the data on screen. if not self.linked.isHidden(): log.info(f'Re-syncing graphics for fsp: {ns_path}') - self.linked.graphics_cycle(trigger_all=True) + self.linked.graphics_cycle( + trigger_all=True, + prepend_update_index=info['first'], + ) else: log.info(f'recved unexpected fsp engine msg: {msg}') From 47cf4aa4f752a677114542eb1669a8268937826c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 May 2022 09:22:46 -0400 Subject: [PATCH 039/113] Error log brokerd msgs that have `.reqid == None` Relates to the bug discovered in #310, this should avoid out-of-order msgs which do not have a `.reqid` set to be error logged to console. Further, add `pformat()` to kraken logging of ems msging. --- piker/brokers/kraken.py | 10 +++++++++- piker/clearing/_ems.py | 26 ++++++++++++++++++++------ piker/ui/order_mode.py | 4 +++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 30e57b9ee..670eed6f2 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -21,6 +21,7 @@ from contextlib import asynccontextmanager as acm from dataclasses import asdict, field from datetime import datetime +from pprint import pformat from typing import Any, Optional, AsyncIterator, Callable, Union import time @@ -569,7 +570,10 @@ async def handle_order_requests( order: BrokerdOrder async for request_msg in ems_order_stream: - log.info(f'Received order request {request_msg}') + log.info( + 'Received order request:\n' + f'{pformat(request_msg)}' + ) action = request_msg['action'] @@ -628,6 +632,7 @@ async def handle_order_requests( # update the internal pairing of oid to krakens # txid with the new txid that is returned on edit reqid = resp['result']['txid'] + # deliver ack that order has been submitted to broker routing await ems_order_stream.send( BrokerdOrderAck( @@ -788,7 +793,10 @@ async def subscribe(ws: wsproto.WSConnection, token: str): # Get websocket token for authenticated data stream # Assert that a token was actually received. resp = await client.endpoint('GetWebSocketsToken', {}) + + # lol wtf is this.. assert resp['error'] == [] + token = resp['result']['token'] async with ( diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index a5d04f0cc..e00676f26 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -561,7 +561,10 @@ async def translate_and_relay_brokerd_events( name = brokerd_msg['name'] - log.info(f'Received broker trade event:\n{pformat(brokerd_msg)}') + log.info( + f'Received broker trade event:\n' + f'{pformat(brokerd_msg)}' + ) if name == 'position': @@ -613,19 +616,28 @@ async def translate_and_relay_brokerd_events( # packed at submission since we already know it ahead of # time paper = brokerd_msg['broker_details'].get('paper_info') + ext = brokerd_msg['broker_details'].get('external') if paper: # paperboi keeps the ems id up front oid = paper['oid'] - else: + elif ext: # may be an order msg specified as "external" to the # piker ems flow (i.e. generated by some other # external broker backend client (like tws for ib) - ext = brokerd_msg['broker_details'].get('external') - if ext: - log.error(f"External trade event {ext}") + log.error(f"External trade event {ext}") continue + + else: + # something is out of order, we don't have an oid for + # this broker-side message. + log.error( + 'Unknown oid:{oid} for msg:\n' + f'{pformat(brokerd_msg)}' + 'Unable to relay message to client side!?' + ) + else: # check for existing live flow entry entry = book._ems_entries.get(oid) @@ -823,7 +835,9 @@ async def process_client_order_cmds( if reqid: # send cancel to brokerd immediately! - log.info("Submitting cancel for live order {reqid}") + log.info( + f'Submitting cancel for live order {reqid}' + ) await brokerd_order_stream.send(msg.dict()) diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 3e230b71d..a86fe816c 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -873,7 +873,9 @@ async def process_trades_and_update_ui( mode.lines.remove_line(uuid=oid) # each clearing tick is responded individually - elif resp in ('broker_filled',): + elif resp in ( + 'broker_filled', + ): known_order = book._sent_orders.get(oid) if not known_order: From c455df7fa85a599ba0e6d604fc0dfbc5135f8831 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:04:31 -0400 Subject: [PATCH 040/113] Drop legacy step path gen, always slice full data Mostly just dropping old commented code for "step mode" format generation. Always slice the tail part of the input data and move to the new `ms_threshold` in the `pg` profiler' --- piker/ui/_curve.py | 63 ++++++++-------------------------------------- 1 file changed, 10 insertions(+), 53 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 66862086c..88917a0ce 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -272,8 +272,6 @@ def downsample( x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() y = y.flatten() - # presumably? - self._in_ds = True return x, y def update_from_array( @@ -305,7 +303,7 @@ def update_from_array( profiler = profiler or pg.debug.Profiler( msg=f'FastAppendCurve.update_from_array(): `{self._name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) flip_cache = False @@ -342,6 +340,10 @@ def update_from_array( showing_src_data = self._in_ds # should_redraw = False + # by default we only pull data up to the last (current) index + x_out_full = x_out = x[:slice_to_head] + y_out_full = y_out = y[:slice_to_head] + # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. if ( @@ -358,7 +360,7 @@ def update_from_array( vl, vr = view_range # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) zoom_or_append = False last_vr = self._vr @@ -414,16 +416,8 @@ def update_from_array( # self.disable_cache() # flip_cache = True - else: - # if ( - # not view_range - # or self._in_ds - # ): - # by default we only pull data up to the last (current) index - x_out, y_out = x[:slice_to_head], y[:slice_to_head] - - if prepend_length > 0: - should_redraw = True + if prepend_length > 0: + should_redraw = True # check for downsampling conditions if ( @@ -459,30 +453,10 @@ def update_from_array( or new_sample_rate or prepend_length > 0 ): - # if ( - # not view_range - # or self._in_ds - # ): - # # by default we only pull data up to the last (current) index - # x_out, y_out = x[:-1], y[:-1] - - # step mode: draw flat top discrete "step" - # over the index space for each datum. - # if self._step_mode: - # self.disable_cache() - # flip_cache = True - # x_out, y_out = step_path_arrays_from_1d( - # x_out, - # y_out, - # ) - - # # TODO: numba this bish - # profiler('generated step arrays') - if should_redraw: if self.path: - # print(f'CLEARING PATH {self._name}') self.path.clear() + profiler('cleared paths due to `should_redraw=True`') if self.fast_path: self.fast_path.clear() @@ -556,23 +530,6 @@ def update_from_array( new_y = y[-append_length - 2:slice_to_head] profiler('sliced append path') - # if self._step_mode: - # # new_x, new_y = step_path_arrays_from_1d( - # # new_x, - # # new_y, - # # ) - # # # [1:] since we don't need the vertical line normally at - # # # the beginning of the step curve taking the first (x, - # # # y) poing down to the x-axis **because** this is an - # # # appended path graphic. - # # new_x = new_x[1:] - # # new_y = new_y[1:] - - # self.disable_cache() - # flip_cache = True - - # profiler('generated step data') - profiler( f'diffed array input, append_length={append_length}' ) @@ -812,7 +769,7 @@ def paint( profiler = pg.debug.Profiler( msg=f'FastAppendCurve.paint(): `{self._name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) self.prepareGeometryChange() From cfc4198837fb3ed96cb7002f509bb29316de4b5a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:16:17 -0400 Subject: [PATCH 041/113] Use new profiler arg name, add more marks throughout flow update --- piker/ui/_flows.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 707be6836..bf6fc3f31 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -46,7 +46,10 @@ ShmArray, # open_shm_array, ) -from .._profile import pg_profile_enabled, ms_slower_then +from .._profile import ( + pg_profile_enabled, + ms_slower_then, +) from ._ohlc import ( BarItems, gen_qpath, @@ -331,11 +334,13 @@ def update_graphics( ''' - profiler = profiler or pg.debug.Profiler( + # profiler = profiler or pg.debug.Profiler( + profiler = pg.debug.Profiler( msg=f'Flow.update_graphics() for {self.name}', disabled=not pg_profile_enabled(), - gt=ms_slower_then, - delayed=True, + # disabled=False, + ms_threshold=4, + # ms_threshold=ms_slower_then, ) # shm read and slice to view read = ( @@ -591,6 +596,7 @@ def update_graphics( # ``FastAppendCurve`` case: array_key = array_key or self.name uppx = graphics.x_uppx() + profiler('read uppx') if graphics._step_mode and self.gy is None: self._iflat_first = self.shm._first.value @@ -616,6 +622,7 @@ def update_graphics( # start y at origin level self.gy[0, 0] = 0 + profiler('generated step mode data') if graphics._step_mode: ( @@ -631,6 +638,7 @@ def update_graphics( ) il = max(iflat - 1, 0) + profiler('read step mode incr update indices') # check for shm prepend updates since last read. if iflat_first != ishm_first: @@ -650,6 +658,7 @@ def update_graphics( # write newly prepended data to flattened copy self.gy[ishm_first:iflat_first] = y2_prepend self._iflat_first = ishm_first + profiler('prepended step mode history') append_diff = ishm_last - iflat if append_diff: @@ -679,6 +688,7 @@ def update_graphics( x_step = self.gx[ishm_first:ishm_last+2] # shape to 1d x = x_step.reshape(-1) + profiler('sliced step x') y_step = self.gy[ishm_first:ishm_last+2] lasts = self.shm.array[['index', array_key]] @@ -690,7 +700,7 @@ def update_graphics( # s = 6 # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - profiler('sliced step data') + profiler('sliced step y') # do all the same for only in-view data ys_iv = y_step[ivl:ivr+1] @@ -703,7 +713,7 @@ def update_graphics( # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - profiler('flattened ustruct in-view OHLC data') + profiler('sliced in view step data') # legacy full-recompute-everytime method # x, y = ohlc_flatten(array) @@ -738,9 +748,11 @@ def update_graphics( # NOTE: already passed through by display loop? # do_append=uppx < 16, + profiler=profiler, **kwargs ) + profiler('updated step mode curve') # graphics.reset_cache() # print( # f"path br: {graphics.path.boundingRect()}\n", @@ -754,9 +766,9 @@ def update_graphics( y = array[array_key] x_iv = in_view['index'] y_iv = in_view[array_key] + profiler('sliced input arrays') # graphics.draw_last(x, y) - profiler(f'draw last segment {array_key}') graphics.update_from_array( x=x, @@ -769,8 +781,10 @@ def update_graphics( # NOTE: already passed through by display loop? # do_append=uppx < 16, + profiler=profiler, **kwargs ) + profiler(f'`graphics.update_from_array()` complete') return graphics From 5e602214bed506b90641e94dcb23c58bb9445c99 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:22:49 -0400 Subject: [PATCH 042/113] Use new flag, add more marks through display loop --- piker/ui/_display.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index d0654b10d..c551fc98c 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -54,10 +54,10 @@ mk_order_pane_layout, ) from .order_mode import open_order_mode -# from .._profile import ( -# pg_profile_enabled, -# ms_slower_then, -# ) +from .._profile import ( + pg_profile_enabled, + ms_slower_then, +) from ..log import get_logger log = get_logger(__name__) @@ -319,9 +319,12 @@ def graphics_update_cycle( profiler = pg.debug.Profiler( msg=f'Graphics loop cycle for: `{chart.name}`', - disabled=True, # not pg_profile_enabled(), - gt=1/12 * 1e3, - # gt=ms_slower_then, + delayed=True, + # disabled=not pg_profile_enabled(), + disabled=True, + ms_threshold=ms_slower_then, + + # ms_threshold=1/12 * 1e3, ) # unpack multi-referenced components @@ -366,7 +369,9 @@ def graphics_update_cycle( l, lbar, rbar, r = brange mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin - profiler('maxmin call') + + profiler('`ds.maxmin()` call') + liv = r >= i_step # the last datum is in view if ( @@ -394,6 +399,7 @@ def graphics_update_cycle( # pixel in a curve should show new data based on uppx # and then iff update curves and shift? chart.increment_view(steps=i_diff) + profiler('view incremented') if vlm_chart: # always update y-label @@ -425,6 +431,7 @@ def graphics_update_cycle( # connected to update accompanying overlay # graphics.. ) + profiler('`vlm_chart.update_graphics_from_flow()`') if ( mx_vlm_in_view != vars['last_mx_vlm'] @@ -433,6 +440,7 @@ def graphics_update_cycle( vlm_chart.view._set_yrange( yrange=yrange, ) + profiler('`vlm_chart.view._set_yrange()`') # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}') vars['last_mx_vlm'] = mx_vlm_in_view From 09e988ec3e1dce0201dbbca903c446b35d5b95af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 13 May 2022 16:23:31 -0400 Subject: [PATCH 043/113] Use `ms_threshold` throughout remaining profilers --- piker/ui/_chart.py | 4 ++-- piker/ui/_interaction.py | 14 ++++++-------- piker/ui/_ohlc.py | 6 +++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8a95327fc..f6fc44ece 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1268,9 +1268,9 @@ def maxmin( ''' profiler = pg.debug.Profiler( - msg=f'`{str(self)}.maxmin()` loop cycle for: `{self.name}`', + msg=f'`{str(self)}.maxmin(name={name})`: `{self.name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, delayed=True, ) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index a2c99e38d..b2b460508 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -762,7 +762,7 @@ def _set_yrange( profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, delayed=True, ) set_range = True @@ -833,7 +833,7 @@ def _set_yrange( ylow, yhigh = yrange - profiler(f'maxmin(): {yrange}') + profiler(f'callback ._maxmin(): {yrange}') # view margins: stay within a % of the "true range" diff = yhigh - ylow @@ -932,8 +932,8 @@ def maybe_downsample_graphics(self): # due to the way delaying works and garbage collection of # the profiler in the delegated method calls. delayed=False, - # gt=3, - gt=ms_slower_then, + ms_threshold=6, + # ms_threshold=ms_slower_then, ) # TODO: a faster single-loop-iterator way of doing this XD @@ -958,9 +958,7 @@ def maybe_downsample_graphics(self): use_vr=True, # gets passed down into graphics obj - profiler=profiler, + # profiler=profiler, ) - profiler(f'range change updated {chart_name}:{name}') - - profiler.finish() + profiler(f'<{chart_name}>.update_graphics_from_flow({name})') diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 328d62b9e..6199b9eab 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -170,7 +170,7 @@ def gen_qpath( profiler = pg.debug.Profiler( msg='gen_qpath ohlc', disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) x, y, c = path_arrays_from_ohlc( @@ -353,7 +353,7 @@ def x_uppx(self) -> int: # ''' # profiler = profiler or pg.debug.Profiler( # disabled=not pg_profile_enabled(), - # gt=ms_slower_then, + # ms_threshold=ms_slower_then, # delayed=True, # ) @@ -718,7 +718,7 @@ def paint( profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), - gt=ms_slower_then, + ms_threshold=ms_slower_then, ) # p.setCompositionMode(0) From 5d2660969354af953f99cdb52c8c319e774eaa6b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 13:36:08 -0400 Subject: [PATCH 044/113] Add "no-tsdb-found" history load length defaults --- piker/data/feed.py | 47 +++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 848fcc108..d5e5d3b39 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -228,7 +228,7 @@ def diff_history( # the + 1 is because ``last_tsdb_dt`` is pulled from # the last row entry for the ``'time'`` field retreived # from the tsdb. - to_push = array[abs(s_diff)+1:] + to_push = array[abs(s_diff) + 1:] else: # pass back only the portion of the array that is @@ -251,6 +251,7 @@ async def start_backfill( last_tsdb_dt: Optional[datetime] = None, storage: Optional[Storage] = None, write_tsdb: bool = True, + tsdb_is_up: bool = False, task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED, @@ -266,8 +267,8 @@ async def start_backfill( # sample period step size in seconds step_size_s = ( - pendulum.from_timestamp(times[-1]) - - pendulum.from_timestamp(times[-2]) + pendulum.from_timestamp(times[-1]) + - pendulum.from_timestamp(times[-2]) ).seconds # "frame"'s worth of sample period steps in seconds @@ -292,25 +293,33 @@ async def start_backfill( # let caller unblock and deliver latest history frame task_status.started((shm, start_dt, end_dt, bf_done)) + # based on the sample step size, maybe load a certain amount history if last_tsdb_dt is None: - # maybe a better default (they don't seem to define epoch?!) - - # based on the sample step size load a certain amount - # history - if step_size_s == 1: - last_tsdb_dt = pendulum.now().subtract(days=2) - - elif step_size_s == 60: - last_tsdb_dt = pendulum.now().subtract(years=2) - - else: + if step_size_s not in (1, 60): raise ValueError( '`piker` only needs to support 1m and 1s sampling ' 'but ur api is trying to deliver a longer ' f'timeframe of {step_size_s} ' 'seconds.. so ye, dun ' - 'do dat bruh.' + 'do dat brudder.' ) + # when no tsdb "last datum" is provided, we just load + # some near-term history. + periods = { + 1: {'days': 1}, + 60: {'days': 14}, + } + + if tsdb_is_up: + # do a decently sized backfill and load it into storage. + periods = { + 1: {'days': 6}, + 60: {'years': 2}, + } + + kwargs = periods[step_size_s] + last_tsdb_dt = start_dt.subtract(**kwargs) + # configure async query throttling erlangs = config.get('erlangs', 1) rate = config.get('rate', 1) @@ -568,8 +577,8 @@ async def get_ohlc_frame( start_dt, end_dt, ) = await get_ohlc_frame( - input_end_dt=last_shm_prepend_dt, - iter_dts_gen=idts, + input_end_dt=last_shm_prepend_dt, + iter_dts_gen=idts, ) last_epoch = to_push['time'][-1] diff = start - last_epoch @@ -1003,7 +1012,7 @@ async def open_feed_bus( brokername: str, symbol: str, # normally expected to the broker-specific fqsn loglevel: str, - tick_throttle: Optional[float] = None, + tick_throttle: Optional[float] = None, start_stream: bool = True, ) -> None: @@ -1264,7 +1273,7 @@ async def search(text: str) -> dict[str, Any]: # a backend module? pause_period=getattr( brokermod, '_search_conf', {} - ).get('pause_period', 0.0616), + ).get('pause_period', 0.0616), ): yield From b609f46d26b80d48e6a8365e40c014e08293d707 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 13:44:56 -0400 Subject: [PATCH 045/113] Always delay interaction update profiling --- piker/ui/_interaction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index b2b460508..90242c999 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -931,7 +931,6 @@ def maybe_downsample_graphics(self): # ``.update_graphics_from_flow()`` nested profiling likely # due to the way delaying works and garbage collection of # the profiler in the delegated method calls. - delayed=False, ms_threshold=6, # ms_threshold=ms_slower_then, ) From f6909ae3952e91bc0e60e96dba67b0f7925e7694 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:12:09 -0400 Subject: [PATCH 046/113] Drop legacy step mode data formatter --- piker/ui/_curve.py | 74 ---------------------------------------------- 1 file changed, 74 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 88917a0ce..d038f085c 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -44,80 +44,6 @@ log = get_logger(__name__) -# TODO: numba this instead.. -# def step_path_arrays_from_1d( -# x: np.ndarray, -# y: np.ndarray, -# include_endpoints: bool = True, - -# ) -> (np.ndarray, np.ndarray): -# ''' -# Generate a "step mode" curve aligned with OHLC style bars -# such that each segment spans each bar (aka "centered" style). - -# ''' -# # y_out = y.copy() -# # x_out = x.copy() - -# # x2 = np.empty( -# # # the data + 2 endpoints on either end for -# # # "termination of the path". -# # (len(x) + 1, 2), -# # # we want to align with OHLC or other sampling style -# # # bars likely so we need fractinal values -# # dtype=float, -# # ) - -# x2 = np.broadcast_to( -# x[:, None], -# ( -# x.size + 1, -# # 4, # only ohlc -# 2, -# ), -# ) + np.array([-0.5, 0.5]) - -# # x2[0] = x[0] - 0.5 -# # x2[1] = x[0] + 0.5 -# # x2[0, 0] = x[0] - 0.5 -# # x2[0, 1] = x[0] + 0.5 -# # x2[1:] = x[:, np.newaxis] + 0.5 -# # import pdbpp -# # pdbpp.set_trace() - -# # flatten to 1-d -# # x_out = x2.reshape(x2.size) -# # x_out = x2 - -# # we create a 1d with 2 extra indexes to -# # hold the start and (current) end value for the steps -# # on either end -# y2 = np.empty( -# (len(y) + 1, 2), -# dtype=y.dtype, -# ) -# y2[:] = y[:, np.newaxis] -# # y2[-1] = 0 - -# # y_out = y2 - -# # y_out = np.empty( -# # 2*len(y) + 2, -# # dtype=y.dtype -# # ) - -# # flatten and set 0 endpoints -# # y_out[1:-1] = y2.reshape(y2.size) -# # y_out[0] = 0 -# # y_out[-1] = 0 - -# if not include_endpoints: -# return x2[:-1], y2[:-1] - -# else: -# return x2, y2 - - _line_styles: dict[str, int] = { 'solid': Qt.PenStyle.SolidLine, 'dash': Qt.PenStyle.DashLine, From e8e26232ea87359c528cdef016a55933e5d54de1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:29:03 -0400 Subject: [PATCH 047/113] Drop `BarItems.update_from_array()`; moved into `Flow` --- piker/ui/_ohlc.py | 283 ---------------------------------------------- 1 file changed, 283 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 6199b9eab..c8da5ba58 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -325,289 +325,6 @@ def x_uppx(self) -> int: else: return 0 - # def update_from_array( - # self, - - # # full array input history - # ohlc: np.ndarray, - - # # pre-sliced array data that's "in view" - # ohlc_iv: np.ndarray, - - # view_range: Optional[tuple[int, int]] = None, - # profiler: Optional[pg.debug.Profiler] = None, - - # ) -> None: - # ''' - # Update the last datum's bar graphic from input data array. - - # This routine should be interface compatible with - # ``pg.PlotCurveItem.setData()``. Normally this method in - # ``pyqtgraph`` seems to update all the data passed to the - # graphics object, and then update/rerender, but here we're - # assuming the prior graphics havent changed (OHLC history rarely - # does) so this "should" be simpler and faster. - - # This routine should be made (transitively) as fast as possible. - - # ''' - # profiler = profiler or pg.debug.Profiler( - # disabled=not pg_profile_enabled(), - # ms_threshold=ms_slower_then, - # delayed=True, - # ) - - # # index = self.start_index - # istart, istop = self._xrange - # # ds_istart, ds_istop = self._ds_xrange - - # index = ohlc['index'] - # first_index, last_index = index[0], index[-1] - - # # length = len(ohlc) - # # prepend_length = istart - first_index - # # append_length = last_index - istop - - # # ds_prepend_length = ds_istart - first_index - # # ds_append_length = last_index - ds_istop - - # flip_cache = False - - # x_gt = 16 - # if self._ds_line: - # uppx = self._ds_line.x_uppx() - # else: - # uppx = 0 - - # should_line = self._in_ds - # if ( - # self._in_ds - # and uppx < x_gt - # ): - # should_line = False - - # elif ( - # not self._in_ds - # and uppx >= x_gt - # ): - # should_line = True - - # profiler('ds logic complete') - - # if should_line: - # # update the line graphic - # # x, y = self._ds_line_xy = ohlc_flatten(ohlc_iv) - # x, y = self._ds_line_xy = ohlc_flatten(ohlc) - # x_iv, y_iv = self._ds_line_xy = ohlc_flatten(ohlc_iv) - # profiler('flattening bars to line') - - # # TODO: we should be diffing the amount of new data which - # # needs to be downsampled. Ideally we actually are just - # # doing all the ds-ing in sibling actors so that the data - # # can just be read and rendered to graphics on events of our - # # choice. - # # diff = do_diff(ohlc, new_bit) - # curve = self._ds_line - # curve.update_from_array( - # x=x, - # y=y, - # x_iv=x_iv, - # y_iv=y_iv, - # view_range=None, # hack - # profiler=profiler, - # ) - # profiler('updated ds line') - - # if not self._in_ds: - # # hide bars and show line - # self.hide() - # # XXX: is this actually any faster? - # # self._pi.removeItem(self) - - # # TODO: a `.ui()` log level? - # log.info( - # f'downsampling to line graphic {self._name}' - # ) - - # # self._pi.addItem(curve) - # curve.show() - # curve.update() - # self._in_ds = True - - # # stop here since we don't need to update bars path any more - # # as we delegate to the downsample line with updates. - - # else: - # # we should be in bars mode - - # if self._in_ds: - # # flip back to bars graphics and hide the downsample line. - # log.info(f'showing bars graphic {self._name}') - - # curve = self._ds_line - # curve.hide() - # # self._pi.removeItem(curve) - - # # XXX: is this actually any faster? - # # self._pi.addItem(self) - # self.show() - # self._in_ds = False - - # # generate in_view path - # self.path = gen_qpath( - # ohlc_iv, - # 0, - # self.w, - # # path=self.path, - # ) - - # # TODO: to make the downsampling faster - # # - allow mapping only a range of lines thus only drawing as - # # many bars as exactly specified. - # # - move ohlc "flattening" to a shmarr - # # - maybe move all this embedded logic to a higher - # # level type? - - # # if prepend_length: - # # # new history was added and we need to render a new path - # # prepend_bars = ohlc[:prepend_length] - - # # if ds_prepend_length: - # # ds_prepend_bars = ohlc[:ds_prepend_length] - # # pre_x, pre_y = ohlc_flatten(ds_prepend_bars) - # # fx = np.concatenate((pre_x, fx)) - # # fy = np.concatenate((pre_y, fy)) - # # profiler('ds line prepend diff complete') - - # # if append_length: - # # # generate new graphics to match provided array - # # # path appending logic: - # # # we need to get the previous "current bar(s)" for the time step - # # # and convert it to a sub-path to append to the historical set - # # # new_bars = ohlc[istop - 1:istop + append_length - 1] - # # append_bars = ohlc[-append_length - 1:-1] - # # # print(f'ohlc bars to append size: {append_bars.size}\n') - - # # if ds_append_length: - # # ds_append_bars = ohlc[-ds_append_length - 1:-1] - # # post_x, post_y = ohlc_flatten(ds_append_bars) - # # print( - # # f'ds curve to append sizes: {(post_x.size, post_y.size)}' - # # ) - # # fx = np.concatenate((fx, post_x)) - # # fy = np.concatenate((fy, post_y)) - - # # profiler('ds line append diff complete') - - # profiler('array diffs complete') - - # # does this work? - # last = ohlc[-1] - # # fy[-1] = last['close'] - - # # # incremental update and cache line datums - # # self._ds_line_xy = fx, fy - - # # maybe downsample to line - # # ds = self.maybe_downsample() - # # if ds: - # # # if we downsample to a line don't bother with - # # # any more path generation / updates - # # self._ds_xrange = first_index, last_index - # # profiler('downsampled to line') - # # return - - # # print(in_view.size) - - # # if self.path: - # # self.path = path - # # self.path.reserve(path.capacity()) - # # self.path.swap(path) - - # # path updates - # # if prepend_length: - # # # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path - # # # y value not matching the first value from - # # # ohlc[prepend_length + 1] ??? - # # prepend_path = gen_qpath(prepend_bars, 0, self.w) - # # old_path = self.path - # # self.path = prepend_path - # # self.path.addPath(old_path) - # # profiler('path PREPEND') - - # # if append_length: - # # append_path = gen_qpath(append_bars, 0, self.w) - - # # self.path.moveTo( - # # float(istop - self.w), - # # float(append_bars[0]['open']) - # # ) - # # self.path.addPath(append_path) - - # # profiler('path APPEND') - # # fp = self.fast_path - # # if fp is None: - # # self.fast_path = append_path - - # # else: - # # fp.moveTo( - # # float(istop - self.w), float(new_bars[0]['open']) - # # ) - # # fp.addPath(append_path) - - # # self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # # flip_cache = True - - # self._xrange = first_index, last_index - - # # trigger redraw despite caching - # self.prepareGeometryChange() - - # self.draw_last(last) - - # # # generate new lines objects for updatable "current bar" - # # self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # # # last bar update - # # i, o, h, l, last, v = last[ - # # ['index', 'open', 'high', 'low', 'close', 'volume'] - # # ] - # # # assert i == self.start_index - 1 - # # # assert i == last_index - # # body, larm, rarm = self._last_bar_lines - - # # # XXX: is there a faster way to modify this? - # # rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # # # writer is responsible for changing open on "first" volume of bar - # # larm.setLine(larm.x1(), o, larm.x2(), o) - - # # if l != h: # noqa - - # # if body is None: - # # body = self._last_bar_lines[0] = QLineF(i, l, i, h) - # # else: - # # # update body - # # body.setLine(i, l, i, h) - - # # # XXX: pretty sure this is causing an issue where the bar has - # # # a large upward move right before the next sample and the body - # # # is getting set to None since the next bar is flat but the shm - # # # array index update wasn't read by the time this code runs. Iow - # # # we're doing this removal of the body for a bar index that is - # # # now out of date / from some previous sample. It's weird - # # # though because i've seen it do this to bars i - 3 back? - - # profiler('last bar set') - - # self.update() - # profiler('.update()') - - # if flip_cache: - # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # # profiler.finish() - def draw_last( self, last: np.ndarray, From bc50db59259d5ec9b59c69c69ff2f73c7eef19e7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 14:30:13 -0400 Subject: [PATCH 048/113] Rename `._ohlc.gen_qpath()` -> `.gen_ohlc_qpath()` --- piker/ui/_flows.py | 4 ++-- piker/ui/_ohlc.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bf6fc3f31..bfbe8aeaf 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -52,7 +52,7 @@ ) from ._ohlc import ( BarItems, - gen_qpath, + gen_ohlc_qpath, ) from ._curve import ( FastAppendCurve, @@ -365,7 +365,7 @@ def update_graphics( r = self._src_r = Renderer( flow=self, # TODO: rename this to something with ohlc - draw_path=gen_qpath, + draw_path=gen_ohlc_qpath, last_read=read, ) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index c8da5ba58..efd50e952 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -156,7 +156,7 @@ def path_arrays_from_ohlc( return x, y, c -def gen_qpath( +def gen_ohlc_qpath( data: np.ndarray, start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap @@ -274,7 +274,7 @@ def draw_from_data( ''' hist, last = ohlc[:-1], ohlc[-1] - self.path = gen_qpath(hist, start, self.w) + self.path = gen_ohlc_qpath(hist, start, self.w) # save graphics for later reference and keep track # of current internal "last index" From 9c5bc6dedaa06c9d42346431b740628cf0e14ec8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:15:14 -0400 Subject: [PATCH 049/113] Add `.ui._pathops` module Starts a module for grouping together all our `QPainterpath` related generation and data format operations for creation of fast curve graphics. To start, drops `FastAppendCurve.downsample()` and moves it to a new `._pathops.xy_downsample()`. --- piker/ui/_curve.py | 40 ++++++----------------------- piker/ui/_flows.py | 31 ---------------------- piker/ui/_pathops.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 63 deletions(-) create mode 100644 piker/ui/_pathops.py diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index d038f085c..4aee9cedf 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -34,10 +34,11 @@ from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor -from ._compression import ( - # ohlc_to_m4_line, - ds_m4, -) +# from ._compression import ( +# # ohlc_to_m4_line, +# ds_m4, +# ) +from ._pathops import xy_downsample from ..log import get_logger @@ -174,32 +175,6 @@ def px_width(self) -> float: QLineF(lbar, 0, rbar, 0) ).length() - def downsample( - self, - x, - y, - px_width, - uppx, - - ) -> tuple[np.ndarray, np.ndarray]: - - # downsample whenever more then 1 pixels per datum can be shown. - # always refresh data bounds until we get diffing - # working properly, see above.. - bins, x, y = ds_m4( - x, - y, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) - ) - x = np.broadcast_to(x[:, None], y.shape) - # x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() - x = (x + np.array([-0.5, 0, 0, 0.5])).flatten() - y = y.flatten() - - return x, y - def update_from_array( self, @@ -396,7 +371,8 @@ def update_from_array( self._in_ds = False elif should_ds and uppx and px_width > 1: - x_out, y_out = self.downsample( + + x_out, y_out = xy_downsample( x_out, y_out, px_width, @@ -461,7 +437,7 @@ def update_from_array( ) # if should_ds: - # new_x, new_y = self.downsample( + # new_x, new_y = xy_downsample( # new_x, # new_y, # px_width, diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bfbe8aeaf..9f70efeaa 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -789,37 +789,6 @@ def update_graphics( return graphics -def xy_downsample( - x, - y, - px_width, - uppx, - - x_spacer: float = 0.5, - -) -> tuple[np.ndarray, np.ndarray]: - - # downsample whenever more then 1 pixels per datum can be shown. - # always refresh data bounds until we get diffing - # working properly, see above.. - bins, x, y = ds_m4( - x, - y, - px_width=px_width, - uppx=uppx, - log_scale=bool(uppx) - ) - - # flatten output to 1d arrays suitable for path-graphics generation. - x = np.broadcast_to(x[:, None], y.shape) - x = (x + np.array( - [-x_spacer, 0, 0, x_spacer] - )).flatten() - y = y.flatten() - - return x, y - - class Renderer(msgspec.Struct): flow: Flow diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py new file mode 100644 index 000000000..654b079ab --- /dev/null +++ b/piker/ui/_pathops.py @@ -0,0 +1,61 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +""" +Super fast ``QPainterPath`` generation related operator routines. + +""" + +import numpy as np +# from numba import njit, float64, int64 # , optional +# import pyqtgraph as pg +# from PyQt5 import QtCore, QtGui, QtWidgets +# from PyQt5.QtCore import QLineF, QPointF + +from ._compression import ( + # ohlc_flatten, + ds_m4, +) + + +def xy_downsample( + x, + y, + px_width, + uppx, + + x_spacer: float = 0.5, + +) -> tuple[np.ndarray, np.ndarray]: + + # downsample whenever more then 1 pixels per datum can be shown. + # always refresh data bounds until we get diffing + # working properly, see above.. + bins, x, y = ds_m4( + x, + y, + px_width=px_width, + uppx=uppx, + # log_scale=bool(uppx) + ) + + # flatten output to 1d arrays suitable for path-graphics generation. + x = np.broadcast_to(x[:, None], y.shape) + x = (x + np.array( + [-x_spacer, 0, 0, x_spacer] + )).flatten() + y = y.flatten() + + return x, y From 037300ced0d254983d6941d30bceafa2120a3f38 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:21:25 -0400 Subject: [PATCH 050/113] Move ohlc lines-curve generators into pathops mod --- piker/ui/_flows.py | 12 ++--- piker/ui/_ohlc.py | 115 +--------------------------------------- piker/ui/_pathops.py | 123 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 124 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 9f70efeaa..38bcf3482 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -48,20 +48,18 @@ ) from .._profile import ( pg_profile_enabled, - ms_slower_then, + # ms_slower_then, +) +from ._pathops import ( + gen_ohlc_qpath, ) from ._ohlc import ( BarItems, - gen_ohlc_qpath, ) from ._curve import ( FastAppendCurve, # step_path_arrays_from_1d, ) -from ._compression import ( - # ohlc_flatten, - ds_m4, -) from ..log import get_logger @@ -784,7 +782,7 @@ def update_graphics( profiler=profiler, **kwargs ) - profiler(f'`graphics.update_from_array()` complete') + profiler('`graphics.update_from_array()` complete') return graphics diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index efd50e952..abe0cb7b3 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -25,7 +25,6 @@ import numpy as np import pyqtgraph as pg -from numba import njit, float64, int64 # , optional from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF # from numba import types as ntypes @@ -36,6 +35,7 @@ from ..log import get_logger from ._curve import FastAppendCurve from ._compression import ohlc_flatten +from ._pathops import gen_ohlc_qpath if TYPE_CHECKING: from ._chart import LinkedSplits @@ -84,119 +84,6 @@ def bar_from_ohlc_row( return [hl, o, c] -@njit( - # TODO: for now need to construct this manually for readonly arrays, see - # https://github.com/numba/numba/issues/4511 - # ntypes.tuple((float64[:], float64[:], float64[:]))( - # numba_ohlc_dtype[::1], # contiguous - # int64, - # optional(float64), - # ), - nogil=True -) -def path_arrays_from_ohlc( - data: np.ndarray, - start: int64, - bar_gap: float64 = 0.43, - -) -> np.ndarray: - ''' - Generate an array of lines objects from input ohlc data. - - ''' - size = int(data.shape[0] * 6) - - x = np.zeros( - # data, - shape=size, - dtype=float64, - ) - y, c = x.copy(), x.copy() - - # TODO: report bug for assert @ - # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991 - for i, q in enumerate(data[start:], start): - - # TODO: ask numba why this doesn't work.. - # open, high, low, close, index = q[ - # ['open', 'high', 'low', 'close', 'index']] - - open = q['open'] - high = q['high'] - low = q['low'] - close = q['close'] - index = float64(q['index']) - - istart = i * 6 - istop = istart + 6 - - # x,y detail the 6 points which connect all vertexes of a ohlc bar - x[istart:istop] = ( - index - bar_gap, - index, - index, - index, - index, - index + bar_gap, - ) - y[istart:istop] = ( - open, - open, - low, - high, - close, - close, - ) - - # specifies that the first edge is never connected to the - # prior bars last edge thus providing a small "gap"/"space" - # between bars determined by ``bar_gap``. - c[istart:istop] = (1, 1, 1, 1, 1, 0) - - return x, y, c - - -def gen_ohlc_qpath( - data: np.ndarray, - start: int = 0, # XXX: do we need this? - # 0.5 is no overlap between arms, 1.0 is full overlap - w: float = 0.43, - path: Optional[QtGui.QPainterPath] = None, - -) -> QtGui.QPainterPath: - - path_was_none = path is None - - profiler = pg.debug.Profiler( - msg='gen_qpath ohlc', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - - x, y, c = path_arrays_from_ohlc( - data, - start, - bar_gap=w, - ) - profiler("generate stream with numba") - - # TODO: numba the internals of this! - path = pg.functions.arrayToQPath( - x, - y, - connect=c, - path=path, - ) - - # avoid mem allocs if possible - if path_was_none: - path.reserve(path.capacity()) - - profiler("generate path with arrayToQPath") - - return path - - class BarItems(pg.GraphicsObject): ''' "Price range" bars graphics rendered from a OHLC sampled sequence. diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 654b079ab..87e3183e3 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -17,13 +17,17 @@ Super fast ``QPainterPath`` generation related operator routines. """ +from typing import ( + Optional, +) import numpy as np -# from numba import njit, float64, int64 # , optional -# import pyqtgraph as pg -# from PyQt5 import QtCore, QtGui, QtWidgets +from numba import njit, float64, int64 # , optional +import pyqtgraph as pg +from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF +from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( # ohlc_flatten, ds_m4, @@ -59,3 +63,116 @@ def xy_downsample( y = y.flatten() return x, y + + +@njit( + # TODO: for now need to construct this manually for readonly arrays, see + # https://github.com/numba/numba/issues/4511 + # ntypes.tuple((float64[:], float64[:], float64[:]))( + # numba_ohlc_dtype[::1], # contiguous + # int64, + # optional(float64), + # ), + nogil=True +) +def path_arrays_from_ohlc( + data: np.ndarray, + start: int64, + bar_gap: float64 = 0.43, + +) -> np.ndarray: + ''' + Generate an array of lines objects from input ohlc data. + + ''' + size = int(data.shape[0] * 6) + + x = np.zeros( + # data, + shape=size, + dtype=float64, + ) + y, c = x.copy(), x.copy() + + # TODO: report bug for assert @ + # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991 + for i, q in enumerate(data[start:], start): + + # TODO: ask numba why this doesn't work.. + # open, high, low, close, index = q[ + # ['open', 'high', 'low', 'close', 'index']] + + open = q['open'] + high = q['high'] + low = q['low'] + close = q['close'] + index = float64(q['index']) + + istart = i * 6 + istop = istart + 6 + + # x,y detail the 6 points which connect all vertexes of a ohlc bar + x[istart:istop] = ( + index - bar_gap, + index, + index, + index, + index, + index + bar_gap, + ) + y[istart:istop] = ( + open, + open, + low, + high, + close, + close, + ) + + # specifies that the first edge is never connected to the + # prior bars last edge thus providing a small "gap"/"space" + # between bars determined by ``bar_gap``. + c[istart:istop] = (1, 1, 1, 1, 1, 0) + + return x, y, c + + +def gen_ohlc_qpath( + data: np.ndarray, + start: int = 0, # XXX: do we need this? + # 0.5 is no overlap between arms, 1.0 is full overlap + w: float = 0.43, + path: Optional[QtGui.QPainterPath] = None, + +) -> QtGui.QPainterPath: + + path_was_none = path is None + + profiler = pg.debug.Profiler( + msg='gen_qpath ohlc', + disabled=not pg_profile_enabled(), + ms_threshold=ms_slower_then, + ) + + x, y, c = path_arrays_from_ohlc( + data, + start, + bar_gap=w, + ) + profiler("generate stream with numba") + + # TODO: numba the internals of this! + path = pg.functions.arrayToQPath( + x, + y, + connect=c, + path=path, + ) + + # avoid mem allocs if possible + if path_was_none: + path.reserve(path.capacity()) + + profiler("generate path with arrayToQPath") + + return path From ca5a25f9217404d78756121d0015c6a66b8ec209 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:44:19 -0400 Subject: [PATCH 051/113] Drop commented `numba` imports --- piker/ui/_ohlc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index abe0cb7b3..14d5b9263 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -27,8 +27,6 @@ import pyqtgraph as pg from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF -# from numba import types as ntypes -# from ..data._source import numba_ohlc_dtype from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor From 537b725bf3738c2e455638e4683e92151107177f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 15:45:06 -0400 Subject: [PATCH 052/113] Factor ohlc to line data conversion into `._pathops.ohlc_to_line()` --- piker/ui/_flows.py | 51 ++++++++++++++++++++++++-------------------- piker/ui/_pathops.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 38bcf3482..266b3aeb6 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -52,13 +52,13 @@ ) from ._pathops import ( gen_ohlc_qpath, + ohlc_to_line, ) from ._ohlc import ( BarItems, ) from ._curve import ( FastAppendCurve, - # step_path_arrays_from_1d, ) from ..log import get_logger @@ -426,29 +426,34 @@ def update_graphics( # create a flattened view onto the OHLC array # which can be read as a line-style format shm = self.shm + ( + self._iflat_first, + self._iflat_last, + self.gx, + self.gy, + ) = ohlc_to_line(shm) + + # self.gy = self.shm.ustruct(fields) + # first = self._iflat_first = self.shm._first.value + # last = self._iflat_last = self.shm._last.value + + # # write pushed data to flattened copy + # self.gy[first:last] = rfn.structured_to_unstructured( + # self.shm.array[fields] + # ) - # flat = self.gy = self.shm.unstruct_view(fields) - self.gy = self.shm.ustruct(fields) - first = self._iflat_first = self.shm._first.value - last = self._iflat_last = self.shm._last.value - - # write pushed data to flattened copy - self.gy[first:last] = rfn.structured_to_unstructured( - self.shm.array[fields] - ) - - # generate an flat-interpolated x-domain - self.gx = ( - np.broadcast_to( - shm._array['index'][:, None], - ( - shm._array.size, - # 4, # only ohlc - self.gy.shape[1], - ), - ) + np.array([-0.5, 0, 0, 0.5]) - ) - assert self.gy.any() + # # generate an flat-interpolated x-domain + # self.gx = ( + # np.broadcast_to( + # shm._array['index'][:, None], + # ( + # shm._array.size, + # # 4, # only ohlc + # self.gy.shape[1], + # ), + # ) + np.array([-0.5, 0, 0, 0.5]) + # ) + # assert self.gy.any() # print(f'unstruct diff: {time.time() - start}') # profiler('read unstr view bars to line') diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 87e3183e3..c1ad383cd 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -22,11 +22,15 @@ ) import numpy as np +from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional import pyqtgraph as pg from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF +from ..data._sharedmem import ( + ShmArray, +) from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( # ohlc_flatten, @@ -176,3 +180,49 @@ def gen_ohlc_qpath( profiler("generate path with arrayToQPath") return path + + +def ohlc_to_line( + ohlc_shm: ShmArray, + fields: list[str] = ['open', 'high', 'low', 'close'] + +) -> tuple[ + int, # flattened first index + int, # flattened last index + np.ndarray, + np.ndarray, +]: + ''' + Convert an input struct-array holding OHLC samples into a pair of + flattened x, y arrays with the same size (datums wise) as the source + data. + + ''' + y_out = ohlc_shm.ustruct(fields) + first = ohlc_shm._first.value + last = ohlc_shm._last.value + + # write pushed data to flattened copy + y_out[first:last] = rfn.structured_to_unstructured( + ohlc_shm.array[fields] + ) + + # generate an flat-interpolated x-domain + x_out = ( + np.broadcast_to( + ohlc_shm._array['index'][:, None], + ( + ohlc_shm._array.size, + # 4, # only ohlc + y_out.shape[1], + ), + ) + np.array([-0.5, 0, 0, 0.5]) + ) + assert y_out.any() + + return ( + first, + last, + x_out, + y_out, + ) From 5d294031f298578dc071a73e650b4cf57abe77af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 16:54:50 -0400 Subject: [PATCH 053/113] Factor step format data gen into `to_step_format()` Yet another path ops routine which converts a 1d array into a data format suitable for rendering a "step curve" graphics path (aka a "bar graph" but implemented as a continuous line). Also, factor the `BarItems` rendering logic (which determines whether to render the literal bars lines or a downsampled curve) into a routine `render_baritems()` until we figure out the right abstraction layer for it. --- piker/ui/_flows.py | 516 +++++++++++++++++++++---------------------- piker/ui/_pathops.py | 28 +++ 2 files changed, 282 insertions(+), 262 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 266b3aeb6..70839ca0f 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -53,6 +53,7 @@ from ._pathops import ( gen_ohlc_qpath, ohlc_to_line, + to_step_format, ) from ._ohlc import ( BarItems, @@ -140,6 +141,243 @@ def mk_ohlc_flat_copy( return y +def render_baritems( + flow: Flow, + graphics: BarItems, + read: tuple[ + int, int, np.ndarray, + int, int, np.ndarray, + ], + profiler: pg.debug.Profiler, + **kwargs, + +) -> None: + ''' + Graphics management logic for a ``BarItems`` object. + + Mostly just logic to determine when and how to downsample an OHLC + lines curve into a flattened line graphic and when to display one + graphic or the other. + + TODO: this should likely be moved into some kind of better abstraction + layer, if not a `Renderer` then something just above it? + + ''' + ( + xfirst, xlast, array, + ivl, ivr, in_view, + ) = read + + # if no source data renderer exists create one. + self = flow + r = self._src_r + if not r: + # OHLC bars path renderer + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + draw_path=gen_ohlc_qpath, + last_read=read, + ) + + ds_curve_r = Renderer( + flow=self, + + # just swap in the flat view + # data_t=lambda array: self.gy.array, + last_read=read, + draw_path=partial( + rowarr_to_path, + x_basis=None, + ), + + ) + curve = FastAppendCurve( + name='OHLC', + color=graphics._color, + ) + curve.hide() + self.plot.addItem(curve) + + # baseline "line" downsampled OHLC curve that should + # kick on only when we reach a certain uppx threshold. + self._render_table[0] = ( + ds_curve_r, + curve, + ) + + dsc_r, curve = self._render_table[0] + + # do checks for whether or not we require downsampling: + # - if we're **not** downsampling then we simply want to + # render the bars graphics curve and update.. + # - if insteam we are in a downsamplig state then we to + x_gt = 6 + uppx = curve.x_uppx() + in_line = should_line = curve.isVisible() + if ( + should_line + and uppx < x_gt + ): + print('FLIPPING TO BARS') + should_line = False + + elif ( + not should_line + and uppx >= x_gt + ): + print('FLIPPING TO LINE') + should_line = True + + profiler(f'ds logic complete line={should_line}') + + # do graphics updates + if should_line: + + fields = ['open', 'high', 'low', 'close'] + if self.gy is None: + # create a flattened view onto the OHLC array + # which can be read as a line-style format + shm = self.shm + ( + self._iflat_first, + self._iflat_last, + self.gx, + self.gy, + ) = ohlc_to_line( + shm, + fields=fields, + ) + + # print(f'unstruct diff: {time.time() - start}') + + gy = self.gy + + # update flatted ohlc copy + ( + iflat_first, + iflat, + ishm_last, + ishm_first, + ) = ( + self._iflat_first, + self._iflat_last, + self.shm._last.value, + self.shm._first.value + ) + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + # write newly prepended data to flattened copy + gy[ + ishm_first:iflat_first + ] = rfn.structured_to_unstructured( + self.shm._array[fields][ishm_first:iflat_first] + ) + self._iflat_first = ishm_first + + to_update = rfn.structured_to_unstructured( + self.shm._array[iflat:ishm_last][fields] + ) + + gy[iflat:ishm_last][:] = to_update + profiler('updated ustruct OHLC data') + + # slice out up-to-last step contents + y_flat = gy[ishm_first:ishm_last] + x_flat = self.gx[ishm_first:ishm_last] + + # update local last-index tracking + self._iflat_last = ishm_last + + # reshape to 1d for graphics rendering + y = y_flat.reshape(-1) + x = x_flat.reshape(-1) + profiler('flattened ustruct OHLC data') + + # do all the same for only in-view data + y_iv_flat = y_flat[ivl:ivr] + x_iv_flat = x_flat[ivl:ivr] + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + profiler('flattened ustruct in-view OHLC data') + + # pass into curve graphics processing + curve.update_from_array( + x, + y, + x_iv=x_iv, + y_iv=y_iv, + view_range=(ivl, ivr), # hack + profiler=profiler, + # should_redraw=False, + + # NOTE: already passed through by display loop? + # do_append=uppx < 16, + **kwargs, + ) + curve.show() + profiler('updated ds curve') + + else: + # render incremental or in-view update + # and apply ouput (path) to graphics. + path, last = r.render( + read, + only_in_view=True, + ) + + graphics.path = path + graphics.draw_last(last) + + # NOTE: on appends we used to have to flip the coords + # cache thought it doesn't seem to be required any more? + # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) + # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # graphics.prepareGeometryChange() + graphics.update() + + if ( + not in_line + and should_line + ): + # change to line graphic + + log.info( + f'downsampling to line graphic {self.name}' + ) + graphics.hide() + # graphics.update() + curve.show() + curve.update() + + elif in_line and not should_line: + log.info(f'showing bars graphic {self.name}') + curve.hide() + graphics.show() + graphics.update() + + # update our pre-downsample-ready data and then pass that + # new data the downsampler algo for incremental update. + + # graphics.update_from_array( + # array, + # in_view, + # view_range=(ivl, ivr) if use_vr else None, + + # **kwargs, + # ) + + # generate and apply path to graphics obj + # graphics.path, last = r.render( + # read, + # only_in_view=True, + # ) + # graphics.draw_last(last) + + class Flow(msgspec.Struct): # , frozen=True): ''' (Financial Signal-)Flow compound type which wraps a real-time @@ -355,276 +593,30 @@ def update_graphics( graphics = self.graphics if isinstance(graphics, BarItems): - - # if no source data renderer exists create one. - r = self._src_r - if not r: - # OHLC bars path renderer - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - draw_path=gen_ohlc_qpath, - last_read=read, - ) - - ds_curve_r = Renderer( - flow=self, - - # just swap in the flat view - # data_t=lambda array: self.gy.array, - last_read=read, - draw_path=partial( - rowarr_to_path, - x_basis=None, - ), - - ) - curve = FastAppendCurve( - name='OHLC', - color=graphics._color, - ) - curve.hide() - self.plot.addItem(curve) - - # baseline "line" downsampled OHLC curve that should - # kick on only when we reach a certain uppx threshold. - self._render_table[0] = ( - ds_curve_r, - curve, - ) - - dsc_r, curve = self._render_table[0] - - # do checks for whether or not we require downsampling: - # - if we're **not** downsampling then we simply want to - # render the bars graphics curve and update.. - # - if insteam we are in a downsamplig state then we to - x_gt = 6 - uppx = curve.x_uppx() - in_line = should_line = curve.isVisible() - if ( - should_line - and uppx < x_gt - ): - print('FLIPPING TO BARS') - should_line = False - - elif ( - not should_line - and uppx >= x_gt - ): - print('FLIPPING TO LINE') - should_line = True - - profiler(f'ds logic complete line={should_line}') - - # do graphics updates - if should_line: - - fields = ['open', 'high', 'low', 'close'] - if self.gy is None: - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - ( - self._iflat_first, - self._iflat_last, - self.gx, - self.gy, - ) = ohlc_to_line(shm) - - # self.gy = self.shm.ustruct(fields) - # first = self._iflat_first = self.shm._first.value - # last = self._iflat_last = self.shm._last.value - - # # write pushed data to flattened copy - # self.gy[first:last] = rfn.structured_to_unstructured( - # self.shm.array[fields] - # ) - - # # generate an flat-interpolated x-domain - # self.gx = ( - # np.broadcast_to( - # shm._array['index'][:, None], - # ( - # shm._array.size, - # # 4, # only ohlc - # self.gy.shape[1], - # ), - # ) + np.array([-0.5, 0, 0, 0.5]) - # ) - # assert self.gy.any() - - # print(f'unstruct diff: {time.time() - start}') - # profiler('read unstr view bars to line') - # start = self.gy._first.value - # update flatted ohlc copy - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value - ) - - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - - # write newly prepended data to flattened copy - self.gy[ - ishm_first:iflat_first - ] = rfn.structured_to_unstructured( - self.shm._array[fields][ishm_first:iflat_first] - ) - self._iflat_first = ishm_first - - # # flat = self.gy = self.shm.unstruct_view(fields) - # self.gy = self.shm.ustruct(fields) - # # self._iflat_last = self.shm._last.value - - # # self._iflat_first = self.shm._first.value - # # do an update for the most recent prepend - # # index - # iflat = ishm_first - - to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm_last][fields] - ) - - self.gy[iflat:ishm_last][:] = to_update - profiler('updated ustruct OHLC data') - - # slice out up-to-last step contents - y_flat = self.gy[ishm_first:ishm_last] - x_flat = self.gx[ishm_first:ishm_last] - - # update local last-index tracking - self._iflat_last = ishm_last - - # reshape to 1d for graphics rendering - y = y_flat.reshape(-1) - x = x_flat.reshape(-1) - profiler('flattened ustruct OHLC data') - - # do all the same for only in-view data - y_iv_flat = y_flat[ivl:ivr] - x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) - x_iv = x_iv_flat.reshape(-1) - profiler('flattened ustruct in-view OHLC data') - - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - - curve.update_from_array( - x, - y, - x_iv=x_iv, - y_iv=y_iv, - view_range=(ivl, ivr), # hack - profiler=profiler, - # should_redraw=False, - - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - **kwargs, - ) - curve.show() - profiler('updated ds curve') - - else: - # render incremental or in-view update - # and apply ouput (path) to graphics. - path, last = r.render( - read, - only_in_view=True, - ) - - graphics.path = path - graphics.draw_last(last) - - # NOTE: on appends we used to have to flip the coords - # cache thought it doesn't seem to be required any more? - # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # graphics.prepareGeometryChange() - graphics.update() - - if ( - not in_line - and should_line - ): - # change to line graphic - - log.info( - f'downsampling to line graphic {self.name}' - ) - graphics.hide() - # graphics.update() - curve.show() - curve.update() - - elif in_line and not should_line: - log.info(f'showing bars graphic {self.name}') - curve.hide() - graphics.show() - graphics.update() - - # update our pre-downsample-ready data and then pass that - # new data the downsampler algo for incremental update. - - # graphics.update_from_array( - # array, - # in_view, - # view_range=(ivl, ivr) if use_vr else None, - - # **kwargs, - # ) - - # generate and apply path to graphics obj - # graphics.path, last = r.render( - # read, - # only_in_view=True, - # ) - # graphics.draw_last(last) + render_baritems( + self, + graphics, + read, + profiler, + **kwargs, + ) else: # ``FastAppendCurve`` case: array_key = array_key or self.name uppx = graphics.x_uppx() - profiler('read uppx') + profiler(f'read uppx {uppx}') if graphics._step_mode and self.gy is None: - self._iflat_first = self.shm._first.value - - # create a flattened view onto the OHLC array - # which can be read as a line-style format shm = self.shm - - # fields = ['index', array_key] - i = shm._array['index'].copy() - out = shm._array[array_key].copy() - - self.gx = np.broadcast_to( - i[:, None], - (i.size, 2), - ) + np.array([-0.5, 0.5]) - - # self.gy = np.broadcast_to( - # out[:, None], (out.size, 2), - # ) - self.gy = np.empty((len(out), 2), dtype=out.dtype) - self.gy[:] = out[:, np.newaxis] - - # start y at origin level - self.gy[0, 0] = 0 + ( + self._iflat_first, + self.gx, + self.gy, + ) = to_step_format( + shm, + array_key, + ) profiler('generated step mode data') if graphics._step_mode: diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index c1ad383cd..2f4913657 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -226,3 +226,31 @@ def ohlc_to_line( x_out, y_out, ) + + +def to_step_format( + shm: ShmArray, + data_field: str, + index_field: str = 'index', + +) -> tuple[int, np.ndarray, np.ndarray]: + ''' + Convert an input 1d shm array to a "step array" format + for use by path graphics generation. + + ''' + first = shm._first.value + i = shm._array['index'].copy() + out = shm._array[data_field].copy() + + x_out = np.broadcast_to( + i[:, None], + (i.size, 2), + ) + np.array([-0.5, 0.5]) + + y_out = np.empty((len(out), 2), dtype=out.dtype) + y_out[:] = out[:, np.newaxis] + + # start y at origin level + y_out[0, 0] = 0 + return first, x_out, y_out From 27ee9fdc81fa5d72e4bb6c596b124a590b2758c9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 15 May 2022 17:06:52 -0400 Subject: [PATCH 054/113] Drop old non-working flatten routine --- piker/ui/_flows.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 70839ca0f..d7ebe4e6d 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -113,34 +113,6 @@ def rowarr_to_path( ) -def mk_ohlc_flat_copy( - ohlc_shm: ShmArray, - - # XXX: we bind this in currently.. - # x_basis: np.ndarray, - - # vr: Optional[slice] = None, - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Return flattened-non-copy view into an OHLC shm array. - - ''' - ohlc = ohlc_shm._array[['open', 'high', 'low', 'close']] - # if vr: - # ohlc = ohlc[vr] - # x = x_basis[vr] - - unstructured = rfn.structured_to_unstructured( - ohlc, - copy=False, - ) - # breakpoint() - y = unstructured.flatten() - # x = x_basis[:y.size] - return y - - def render_baritems( flow: Flow, graphics: BarItems, From b236dc72e441c190f71119999d4dd441fc78d234 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 14:31:04 -0400 Subject: [PATCH 055/113] Make vlm a float; discrete is so 80s --- piker/data/_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/data/_source.py b/piker/data/_source.py index 2f5f61ed9..9afcb191c 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -33,7 +33,7 @@ ('high', float), ('low', float), ('close', float), - ('volume', int), + ('volume', float), ('bar_wap', float), ] From 1dca7766d20763f3eeae7c735cf8f9843af875c9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 14:31:23 -0400 Subject: [PATCH 056/113] Add notes about how to do mkts "trimming" Which is basically just "deleting" rows from a column series. You can only use the trim command from the `.cmd` cli and only with a so called `LocalClient` currently; it's also sketchy af and caused a machine to hang due to mem usage.. Ideally we can patch in this functionality for use by the rpc api and have it not hang like this XD Pertains to https://github.com/alpacahq/marketstore/issues/264 --- piker/data/marketstore.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index e1fb38d5d..43b156718 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -230,8 +230,8 @@ def start_marketstore( # ohlcv sampling ('Open', 'f4'), ('High', 'f4'), - ('Low', 'i8'), - ('Close', 'i8'), + ('Low', 'f4'), + ('Close', 'f4'), ('Volume', 'f4'), ] @@ -547,6 +547,17 @@ async def write_ohlcv( if err: raise MarketStoreError(err) + # XXX: currently the only way to do this is through the CLI: + + # sudo ./marketstore connect --dir ~/.config/piker/data + # >> \show mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15 + # and this seems to block and use up mem.. + # >> \trim mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15 + + # relevant source code for this is here: + # https://github.com/alpacahq/marketstore/blob/master/cmd/connect/session/trim.go#L14 + # def delete_range(self, start_dt, end_dt) -> None: + # ... @acm async def open_storage_client( From 1f95ba4fd81d2bfceafe599eed15dffeb1a0ead4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 17:58:44 -0400 Subject: [PATCH 057/113] Drop input xy from constructor, only keep state for cursor stuff.. --- piker/ui/_chart.py | 2 -- piker/ui/_curve.py | 23 ++++++++++------------- piker/ui/_ohlc.py | 2 -- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index f6fc44ece..2eba9a24a 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1075,8 +1075,6 @@ def draw_curve( # yah, we wrote our own B) data = shm.array curve = FastAppendCurve( - y=data[data_key], - x=data['index'], # antialias=True, name=name, diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4aee9cedf..b404cb767 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -68,9 +68,6 @@ class FastAppendCurve(pg.GraphicsObject): ''' def __init__( self, - - x: np.ndarray = None, - y: np.ndarray = None, *args, step_mode: bool = False, @@ -85,8 +82,8 @@ def __init__( ) -> None: # brutaaalll, see comments within.. - self._y = self.yData = y - self._x = self.xData = x + self.yData = None + self.xData = None self._vr: Optional[tuple] = None self._avr: Optional[tuple] = None self._br = None @@ -206,7 +203,7 @@ def update_from_array( disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) - flip_cache = False + # flip_cache = False if self._xrange: istart, istop = self._xrange @@ -227,9 +224,8 @@ def update_from_array( # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. - # self.xData = x - # self.yData = y - # self._x, self._y = x, y + self.xData = x + self.yData = y # downsampling incremental state checking uppx = self.x_uppx() @@ -261,7 +257,7 @@ def update_from_array( vl, vr = view_range # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) zoom_or_append = False last_vr = self._vr @@ -390,7 +386,9 @@ def update_from_array( ) self.prepareGeometryChange() profiler( - f'generated fresh path. (should_redraw: {should_redraw} should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + 'generated fresh path. ' + f'(should_redraw: {should_redraw} ' + f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) # profiler(f'DRAW PATH IN VIEW -> {self._name}') @@ -495,7 +493,6 @@ def update_from_array( self.draw_last(x, y) profiler('draw last segment') - # if flip_cache: # # # XXX: seems to be needed to avoid artifacts (see above). # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) @@ -545,7 +542,7 @@ def draw_last( # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): - return self._x, self._y + return self.xData, self.yData # TODO: drop the above after ``Cursor`` re-work def get_arrays(self) -> tuple[np.ndarray, np.ndarray]: diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 14d5b9263..88fa62f9e 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -187,8 +187,6 @@ def draw_from_data( # curve that does not release mem allocs: # https://doc.qt.io/qt-5/qpainterpath.html#clear curve = FastAppendCurve( - y=y, - x=x, name='OHLC', color=self._color, ) From f67fd11a29d5b5eefe46566b57ded4adfb74b698 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 16 May 2022 17:59:10 -0400 Subject: [PATCH 058/113] Little formattito --- piker/ui/_cursor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index 8f18fe458..fe5fc100a 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -98,9 +98,11 @@ def event( ev: QtCore.QEvent, ) -> bool: - if not isinstance( - ev, QtCore.QDynamicPropertyChangeEvent - ) or self.curve() is None: + + if ( + not isinstance(ev, QtCore.QDynamicPropertyChangeEvent) + or self.curve() is None + ): return False # TODO: get rid of this ``.getData()`` and @@ -115,7 +117,10 @@ def event( i = round(index - x[0]) if i > 0 and i < len(y): newPos = (index, y[i]) - QtWidgets.QGraphicsItem.setPos(self, *newPos) + QtWidgets.QGraphicsItem.setPos( + self, + *newPos, + ) return True return False From df1c89e8118662199131f1fcce4811e09af9fef1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:06:57 -0400 Subject: [PATCH 059/113] Drop all "pixel width" refs (`px_width`) from m4 impl --- piker/ui/_compression.py | 85 ++++++---------------------------------- piker/ui/_pathops.py | 5 +-- 2 files changed, 13 insertions(+), 77 deletions(-) diff --git a/piker/ui/_compression.py b/piker/ui/_compression.py index 5e8b759a4..e9564359b 100644 --- a/piker/ui/_compression.py +++ b/piker/ui/_compression.py @@ -138,51 +138,20 @@ def ohlc_flatten( return x, flat -def ohlc_to_m4_line( - ohlc: np.ndarray, - px_width: int, - - downsample: bool = False, - uppx: Optional[float] = None, - pretrace: bool = False, - -) -> tuple[np.ndarray, np.ndarray]: - ''' - Convert an OHLC struct-array to a m4 downsampled 1-d array. - - ''' - xpts, flat = ohlc_flatten( - ohlc, - use_mxmn=pretrace, - ) - - if downsample: - bins, x, y = ds_m4( - xpts, - flat, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) - ) - x = np.broadcast_to(x[:, None], y.shape) - x = (x + np.array([-0.43, 0, 0, 0.43])).flatten() - y = y.flatten() - - return x, y - else: - return xpts, flat - - def ds_m4( x: np.ndarray, y: np.ndarray, + # units-per-pixel-x(dimension) + uppx: float, + + # XXX: troll zone / easter egg.. + # want to mess with ur pal, pass in the actual + # pixel width here instead of uppx-proper (i.e. pass + # in our ``pg.GraphicsObject`` derivative's ``.px_width()`` + # gto mega-trip-out ur bud). Hint, it used to be implemented + # (wrongly) using "pixel width", so check the git history ;) - # this is the width of the data in view - # in display-device-local pixel units. - px_width: int, - uppx: Optional[float] = None, xrange: Optional[float] = None, - # log_scale: bool = True, ) -> tuple[int, np.ndarray, np.ndarray]: ''' @@ -209,29 +178,8 @@ def ds_m4( # "i didn't show it in the sample code, but it's accounted for # in the start and end indices and number of bins" - # optionally log-scale down the "supposed pxs on screen" - # as the units-per-px (uppx) get's large. - # if log_scale: - # assert uppx, 'You must provide a `uppx` value to use log scaling!' - # # uppx = uppx * math.log(uppx, 2) - - # # scaler = 2**7 / (1 + math.log(uppx, 2)) - # scaler = round( - # max( - # # NOTE: found that a 16x px width brought greater - # # detail, likely due to dpi scaling? - # # px_width=px_width * 16, - # 2**7 / (1 + math.log(uppx, 2)), - # 1 - # ) - # ) - # px_width *= scaler - - # else: - # px_width *= 16 - # should never get called unless actually needed - assert px_width > 1 and uppx > 0 + assert uppx > 1 # NOTE: if we didn't pre-slice the data to downsample # you could in theory pass these as the slicing params, @@ -248,16 +196,9 @@ def ds_m4( # uppx *= max(4 / (1 + math.log(uppx, 2)), 1) pxw = math.ceil(xrange / uppx) - # px_width = math.ceil(px_width) - - # ratio of indexed x-value to width of raster in pixels. - # this is more or less, uppx: units-per-pixel. - # w = xrange / float(px_width) - # uppx = uppx * math.log(uppx, 2) - # w2 = px_width / uppx - # scale up the width as the uppx get's large - w = uppx # * math.log(uppx, 666) + # scale up the frame "width" directly with uppx + w = uppx # ensure we make more then enough # frames (windows) for the output pixel @@ -276,9 +217,7 @@ def ds_m4( # print( # f'uppx: {uppx}\n' # f'xrange: {xrange}\n' - # f'px_width: {px_width}\n' # f'pxw: {pxw}\n' - # f'WTF w:{w}, w2:{w2}\n' # f'frames: {frames}\n' # ) assert frames >= (xrange / uppx) diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 2f4913657..f7eaf2a72 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -41,7 +41,6 @@ def xy_downsample( x, y, - px_width, uppx, x_spacer: float = 0.5, @@ -54,9 +53,7 @@ def xy_downsample( bins, x, y = ds_m4( x, y, - px_width=px_width, - uppx=uppx, - # log_scale=bool(uppx) + uppx, ) # flatten output to 1d arrays suitable for path-graphics generation. From 81be0b4bd0b3b8e3e65a5552df992317d0ac221e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:14:49 -0400 Subject: [PATCH 060/113] Dont pass `px_width` to m4, add some commented path cap tracking --- piker/ui/_curve.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index b404cb767..8fd311991 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -86,7 +86,7 @@ def __init__( self.xData = None self._vr: Optional[tuple] = None self._avr: Optional[tuple] = None - self._br = None + self._last_cap: int = 0 self._name = name self.path: Optional[QtGui.QPainterPath] = None @@ -238,8 +238,8 @@ def update_from_array( # should_redraw = False # by default we only pull data up to the last (current) index - x_out_full = x_out = x[:slice_to_head] - y_out_full = y_out = y[:slice_to_head] + x_out = x[:slice_to_head] + y_out = y[:slice_to_head] # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. @@ -319,8 +319,7 @@ def update_from_array( # check for downsampling conditions if ( # std m4 downsample conditions - px_width - and abs(uppx_diff) >= 1 + abs(uppx_diff) >= 1 ): log.info( f'{self._name} sampler change: {self._last_uppx} -> {uppx}' @@ -366,12 +365,11 @@ def update_from_array( self._in_ds = False - elif should_ds and uppx and px_width > 1: + elif should_ds and uppx > 1: x_out, y_out = xy_downsample( x_out, y_out, - px_width, uppx, ) profiler(f'FULL PATH downsample redraw={should_ds}') @@ -438,7 +436,6 @@ def update_from_array( # new_x, new_y = xy_downsample( # new_x, # new_y, - # px_width, # uppx, # ) # profiler(f'fast path downsample redraw={should_ds}') @@ -489,9 +486,9 @@ def update_from_array( # self.disable_cache() # flip_cache = True - if draw_last: - self.draw_last(x, y) - profiler('draw last segment') + # if draw_last: + # self.draw_last(x, y) + # profiler('draw last segment') # if flip_cache: # # # XXX: seems to be needed to avoid artifacts (see above). @@ -544,10 +541,6 @@ def draw_last( def getData(self): return self.xData, self.yData - # TODO: drop the above after ``Cursor`` re-work - def get_arrays(self) -> tuple[np.ndarray, np.ndarray]: - return self._x, self._y - def clear(self): ''' Clear internal graphics making object ready for full re-draw. @@ -653,7 +646,6 @@ def _path_br(self): # hb_size, QSizeF(w, h) ) - self._br = br # print(f'bounding rect: {br}') return br @@ -691,6 +683,11 @@ def paint( path = self.path + # cap = path.capacity() + # if cap != self._last_cap: + # print(f'NEW CAPACITY: {self._last_cap} -> {cap}') + # self._last_cap = cap + if path: p.drawPath(path) profiler(f'.drawPath(path): {path.capacity()}') From e258654c862a93dfc7bcbeace9bb617e1d1a2ad7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 17 May 2022 19:18:31 -0400 Subject: [PATCH 061/113] Just drop "line dot" updates for now.. --- piker/ui/_cursor.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/piker/ui/_cursor.py b/piker/ui/_cursor.py index fe5fc100a..606ff3f24 100644 --- a/piker/ui/_cursor.py +++ b/piker/ui/_cursor.py @@ -108,20 +108,20 @@ def event( # TODO: get rid of this ``.getData()`` and # make a more pythonic api to retreive backing # numpy arrays... - (x, y) = self.curve().getData() - index = self.property('index') - # first = self._plot._arrays['ohlc'][0]['index'] - # first = x[0] - # i = index - first - if index: - i = round(index - x[0]) - if i > 0 and i < len(y): - newPos = (index, y[i]) - QtWidgets.QGraphicsItem.setPos( - self, - *newPos, - ) - return True + # (x, y) = self.curve().getData() + # index = self.property('index') + # # first = self._plot._arrays['ohlc'][0]['index'] + # # first = x[0] + # # i = index - first + # if index: + # i = round(index - x[0]) + # if i > 0 and i < len(y): + # newPos = (index, y[i]) + # QtWidgets.QGraphicsItem.setPos( + # self, + # *newPos, + # ) + # return True return False From 4c7661fc2339e59aa4204f34f70002129dd4d648 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:24:12 -0400 Subject: [PATCH 062/113] Factor `.update_from_array()` into `Flow.update_graphics()` A bit hacky to get all graphics types working but this is hopefully the first step toward moving all the generic update logic into `Renderer` types which can be themselves managed more compactly and cached per uppx-m4 level. --- piker/ui/_curve.py | 652 +++++++++++++++++------------------ piker/ui/_flows.py | 829 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 950 insertions(+), 531 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8fd311991..9e1f684ab 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -172,332 +172,332 @@ def px_width(self) -> float: QLineF(lbar, 0, rbar, 0) ).length() - def update_from_array( - self, - - # full array input history - x: np.ndarray, - y: np.ndarray, - - # pre-sliced array data that's "in view" - x_iv: np.ndarray, - y_iv: np.ndarray, - - view_range: Optional[tuple[int, int]] = None, - profiler: Optional[pg.debug.Profiler] = None, - draw_last: bool = True, - slice_to_head: int = -1, - do_append: bool = True, - should_redraw: bool = False, - - ) -> QtGui.QPainterPath: - ''' - Update curve from input 2-d data. - - Compare with a cached "x-range" state and (pre/a)ppend based on - a length diff. - - ''' - profiler = profiler or pg.debug.Profiler( - msg=f'FastAppendCurve.update_from_array(): `{self._name}`', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - # flip_cache = False - - if self._xrange: - istart, istop = self._xrange - else: - self._xrange = istart, istop = x[0], x[-1] - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - prepend_length = int(istart - x[0]) - append_length = int(x[-1] - istop) - - # this is the diff-mode, "data"-rendered index - # tracking var.. - self._xrange = x[0], x[-1] - - # print(f"xrange: {self._xrange}") - - # XXX: lol brutal, the internals of `CurvePoint` (inherited by - # our `LineDot`) required ``.getData()`` to work.. - self.xData = x - self.yData = y - - # downsampling incremental state checking - uppx = self.x_uppx() - px_width = self.px_width() - uppx_diff = (uppx - self._last_uppx) - - new_sample_rate = False - should_ds = self._in_ds - showing_src_data = self._in_ds - # should_redraw = False - - # by default we only pull data up to the last (current) index - x_out = x[:slice_to_head] - y_out = y[:slice_to_head] - - # if a view range is passed, plan to draw the - # source ouput that's "in view" of the chart. - if ( - view_range - # and not self._in_ds - # and not prepend_length > 0 - ): - # print(f'{self._name} vr: {view_range}') - - # by default we only pull data up to the last (current) index - x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] - profiler(f'view range slice {view_range}') - - vl, vr = view_range - - # last_ivr = self._x_iv_range - # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) - - zoom_or_append = False - last_vr = self._vr - last_ivr = self._avr - - if last_vr: - # relative slice indices - lvl, lvr = last_vr - # abs slice indices - al, ar = last_ivr - - # append_length = int(x[-1] - istop) - # append_length = int(x_iv[-1] - ar) - - # left_change = abs(x_iv[0] - al) >= 1 - # right_change = abs(x_iv[-1] - ar) >= 1 - - if ( - # likely a zoom view change - (vr - lvr) > 2 or vl < lvl - # append / prepend update - # we had an append update where the view range - # didn't change but the data-viewed (shifted) - # underneath, so we need to redraw. - # or left_change and right_change and last_vr == view_range - - # not (left_change and right_change) and ivr - # ( - # or abs(x_iv[ivr] - livr) > 1 - ): - zoom_or_append = True - - # if last_ivr: - # liivl, liivr = last_ivr - - if ( - view_range != last_vr - and ( - append_length > 1 - or zoom_or_append - ) - ): - should_redraw = True - # print("REDRAWING BRUH") - - self._vr = view_range - self._avr = x_iv[0], x_iv[slice_to_head] - - # x_last = x_iv[-1] - # y_last = y_iv[-1] - # self._last_vr = view_range - - # self.disable_cache() - # flip_cache = True - - if prepend_length > 0: - should_redraw = True - - # check for downsampling conditions - if ( - # std m4 downsample conditions - abs(uppx_diff) >= 1 - ): - log.info( - f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - ) - self._last_uppx = uppx - new_sample_rate = True - showing_src_data = False - should_redraw = True - should_ds = True - - elif ( - uppx <= 2 - and self._in_ds - ): - # we should de-downsample back to our original - # source data so we clear our path data in prep - # to generate a new one from original source data. - should_redraw = True - new_sample_rate = True - should_ds = False - showing_src_data = True - - # no_path_yet = self.path is None - if ( - self.path is None - or should_redraw - or new_sample_rate - or prepend_length > 0 - ): - if should_redraw: - if self.path: - self.path.clear() - profiler('cleared paths due to `should_redraw=True`') - - if self.fast_path: - self.fast_path.clear() - - profiler('cleared paths due to `should_redraw` set') - - if new_sample_rate and showing_src_data: - # if self._in_ds: - log.info(f'DEDOWN -> {self._name}') - - self._in_ds = False - - elif should_ds and uppx > 1: - - x_out, y_out = xy_downsample( - x_out, - y_out, - uppx, - ) - profiler(f'FULL PATH downsample redraw={should_ds}') - self._in_ds = True - - self.path = pg.functions.arrayToQPath( - x_out, - y_out, - connect='all', - finiteCheck=False, - path=self.path, - ) - self.prepareGeometryChange() - profiler( - 'generated fresh path. ' - f'(should_redraw: {should_redraw} ' - f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' - ) - # profiler(f'DRAW PATH IN VIEW -> {self._name}') - - # reserve mem allocs see: - # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # XXX: right now this is based on had hoc checks on a - # hidpi 3840x2160 4k monitor but we should optimize for - # the target display(s) on the sys. - # if no_path_yet: - # self.path.reserve(int(500e3)) - - # TODO: get this piecewise prepend working - right now it's - # giving heck on vwap... - # elif prepend_length: - # breakpoint() - - # prepend_path = pg.functions.arrayToQPath( - # x[0:prepend_length], - # y[0:prepend_length], - # connect='all' - # ) - - # # swap prepend path in "front" - # old_path = self.path - # self.path = prepend_path - # # self.path.moveTo(new_x[0], new_y[0]) - # self.path.connectPath(old_path) - - elif ( - append_length > 0 - and do_append - and not should_redraw - # and not view_range - ): - print(f'{self._name} append len: {append_length}') - new_x = x[-append_length - 2:slice_to_head] - new_y = y[-append_length - 2:slice_to_head] - profiler('sliced append path') - - profiler( - f'diffed array input, append_length={append_length}' - ) - - # if should_ds: - # new_x, new_y = xy_downsample( - # new_x, - # new_y, - # uppx, - # ) - # profiler(f'fast path downsample redraw={should_ds}') - - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - finiteCheck=False, - path=self.fast_path, - ) - profiler('generated append qpath') - - if self.use_fpath: - # an attempt at trying to make append-updates faster.. - if self.fast_path is None: - self.fast_path = append_path - # self.fast_path.reserve(int(6e3)) - else: - self.fast_path.connectPath(append_path) - size = self.fast_path.capacity() - profiler(f'connected fast path w size: {size}') - - # print(f"append_path br: {append_path.boundingRect()}") - # self.path.moveTo(new_x[0], new_y[0]) - # path.connectPath(append_path) - - # XXX: lol this causes a hang.. - # self.path = self.path.simplified() - else: - size = self.path.capacity() - profiler(f'connected history path w size: {size}') - self.path.connectPath(append_path) - - # other merging ideas: - # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths - # path.addPath(append_path) - # path.closeSubpath() - - # TODO: try out new work from `pyqtgraph` main which - # should repair horrid perf: - # https://github.com/pyqtgraph/pyqtgraph/pull/2032 - # ok, nope still horrible XD - # if self._fill: - # # XXX: super slow set "union" op - # self.path = self.path.united(append_path).simplified() - - # self.disable_cache() - # flip_cache = True - - # if draw_last: - # self.draw_last(x, y) - # profiler('draw last segment') - - # if flip_cache: - # # # XXX: seems to be needed to avoid artifacts (see above). - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - # trigger redraw of path - # do update before reverting to cache mode - self.update() - profiler('.update()') + # def update_from_array( + # self, + + # # full array input history + # x: np.ndarray, + # y: np.ndarray, + + # # pre-sliced array data that's "in view" + # x_iv: np.ndarray, + # y_iv: np.ndarray, + + # view_range: Optional[tuple[int, int]] = None, + # profiler: Optional[pg.debug.Profiler] = None, + # draw_last: bool = True, + # slice_to_head: int = -1, + # do_append: bool = True, + # should_redraw: bool = False, + + # ) -> QtGui.QPainterPath: + # ''' + # Update curve from input 2-d data. + + # Compare with a cached "x-range" state and (pre/a)ppend based on + # a length diff. + + # ''' + # profiler = profiler or pg.debug.Profiler( + # msg=f'FastAppendCurve.update_from_array(): `{self._name}`', + # disabled=not pg_profile_enabled(), + # ms_threshold=ms_slower_then, + # ) + # # flip_cache = False + + # if self._xrange: + # istart, istop = self._xrange + # else: + # self._xrange = istart, istop = x[0], x[-1] + + # # compute the length diffs between the first/last index entry in + # # the input data and the last indexes we have on record from the + # # last time we updated the curve index. + # prepend_length = int(istart - x[0]) + # append_length = int(x[-1] - istop) + + # # this is the diff-mode, "data"-rendered index + # # tracking var.. + # self._xrange = x[0], x[-1] + + # # print(f"xrange: {self._xrange}") + + # # XXX: lol brutal, the internals of `CurvePoint` (inherited by + # # our `LineDot`) required ``.getData()`` to work.. + # self.xData = x + # self.yData = y + + # # downsampling incremental state checking + # uppx = self.x_uppx() + # px_width = self.px_width() + # uppx_diff = (uppx - self._last_uppx) + + # new_sample_rate = False + # should_ds = self._in_ds + # showing_src_data = self._in_ds + # # should_redraw = False + + # # by default we only pull data up to the last (current) index + # x_out = x[:slice_to_head] + # y_out = y[:slice_to_head] + + # # if a view range is passed, plan to draw the + # # source ouput that's "in view" of the chart. + # if ( + # view_range + # # and not self._in_ds + # # and not prepend_length > 0 + # ): + # # print(f'{self._name} vr: {view_range}') + + # # by default we only pull data up to the last (current) index + # x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] + # profiler(f'view range slice {view_range}') + + # vl, vr = view_range + + # # last_ivr = self._x_iv_range + # # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) + + # zoom_or_append = False + # last_vr = self._vr + # last_ivr = self._avr + + # if last_vr: + # # relative slice indices + # lvl, lvr = last_vr + # # abs slice indices + # al, ar = last_ivr + + # # append_length = int(x[-1] - istop) + # # append_length = int(x_iv[-1] - ar) + + # # left_change = abs(x_iv[0] - al) >= 1 + # # right_change = abs(x_iv[-1] - ar) >= 1 + + # if ( + # # likely a zoom view change + # (vr - lvr) > 2 or vl < lvl + # # append / prepend update + # # we had an append update where the view range + # # didn't change but the data-viewed (shifted) + # # underneath, so we need to redraw. + # # or left_change and right_change and last_vr == view_range + + # # not (left_change and right_change) and ivr + # # ( + # # or abs(x_iv[ivr] - livr) > 1 + # ): + # zoom_or_append = True + + # # if last_ivr: + # # liivl, liivr = last_ivr + + # if ( + # view_range != last_vr + # and ( + # append_length > 1 + # or zoom_or_append + # ) + # ): + # should_redraw = True + # # print("REDRAWING BRUH") + + # self._vr = view_range + # self._avr = x_iv[0], x_iv[slice_to_head] + + # # x_last = x_iv[-1] + # # y_last = y_iv[-1] + # # self._last_vr = view_range + + # # self.disable_cache() + # # flip_cache = True + + # if prepend_length > 0: + # should_redraw = True + + # # check for downsampling conditions + # if ( + # # std m4 downsample conditions + # abs(uppx_diff) >= 1 + # ): + # log.info( + # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' + # ) + # self._last_uppx = uppx + # new_sample_rate = True + # showing_src_data = False + # should_redraw = True + # should_ds = True + + # elif ( + # uppx <= 2 + # and self._in_ds + # ): + # # we should de-downsample back to our original + # # source data so we clear our path data in prep + # # to generate a new one from original source data. + # should_redraw = True + # new_sample_rate = True + # should_ds = False + # showing_src_data = True + + # # no_path_yet = self.path is None + # if ( + # self.path is None + # or should_redraw + # or new_sample_rate + # or prepend_length > 0 + # ): + # if should_redraw: + # if self.path: + # self.path.clear() + # profiler('cleared paths due to `should_redraw=True`') + + # if self.fast_path: + # self.fast_path.clear() + + # profiler('cleared paths due to `should_redraw` set') + + # if new_sample_rate and showing_src_data: + # # if self._in_ds: + # log.info(f'DEDOWN -> {self._name}') + + # self._in_ds = False + + # elif should_ds and uppx > 1: + + # x_out, y_out = xy_downsample( + # x_out, + # y_out, + # uppx, + # ) + # profiler(f'FULL PATH downsample redraw={should_ds}') + # self._in_ds = True + + # self.path = pg.functions.arrayToQPath( + # x_out, + # y_out, + # connect='all', + # finiteCheck=False, + # path=self.path, + # ) + # self.prepareGeometryChange() + # profiler( + # 'generated fresh path. ' + # f'(should_redraw: {should_redraw} ' + # f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + # ) + # # profiler(f'DRAW PATH IN VIEW -> {self._name}') + + # # reserve mem allocs see: + # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # # XXX: right now this is based on had hoc checks on a + # # hidpi 3840x2160 4k monitor but we should optimize for + # # the target display(s) on the sys. + # # if no_path_yet: + # # self.path.reserve(int(500e3)) + + # # TODO: get this piecewise prepend working - right now it's + # # giving heck on vwap... + # # elif prepend_length: + # # breakpoint() + + # # prepend_path = pg.functions.arrayToQPath( + # # x[0:prepend_length], + # # y[0:prepend_length], + # # connect='all' + # # ) + + # # # swap prepend path in "front" + # # old_path = self.path + # # self.path = prepend_path + # # # self.path.moveTo(new_x[0], new_y[0]) + # # self.path.connectPath(old_path) + + # elif ( + # append_length > 0 + # and do_append + # and not should_redraw + # # and not view_range + # ): + # print(f'{self._name} append len: {append_length}') + # new_x = x[-append_length - 2:slice_to_head] + # new_y = y[-append_length - 2:slice_to_head] + # profiler('sliced append path') + + # profiler( + # f'diffed array input, append_length={append_length}' + # ) + + # # if should_ds: + # # new_x, new_y = xy_downsample( + # # new_x, + # # new_y, + # # uppx, + # # ) + # # profiler(f'fast path downsample redraw={should_ds}') + + # append_path = pg.functions.arrayToQPath( + # new_x, + # new_y, + # connect='all', + # finiteCheck=False, + # path=self.fast_path, + # ) + # profiler('generated append qpath') + + # if self.use_fpath: + # # an attempt at trying to make append-updates faster.. + # if self.fast_path is None: + # self.fast_path = append_path + # # self.fast_path.reserve(int(6e3)) + # else: + # self.fast_path.connectPath(append_path) + # size = self.fast_path.capacity() + # profiler(f'connected fast path w size: {size}') + + # # print(f"append_path br: {append_path.boundingRect()}") + # # self.path.moveTo(new_x[0], new_y[0]) + # # path.connectPath(append_path) + + # # XXX: lol this causes a hang.. + # # self.path = self.path.simplified() + # else: + # size = self.path.capacity() + # profiler(f'connected history path w size: {size}') + # self.path.connectPath(append_path) + + # # other merging ideas: + # # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths + # # path.addPath(append_path) + # # path.closeSubpath() + + # # TODO: try out new work from `pyqtgraph` main which + # # should repair horrid perf: + # # https://github.com/pyqtgraph/pyqtgraph/pull/2032 + # # ok, nope still horrible XD + # # if self._fill: + # # # XXX: super slow set "union" op + # # self.path = self.path.united(append_path).simplified() + + # # self.disable_cache() + # # flip_cache = True + + # # if draw_last: + # # self.draw_last(x, y) + # # profiler('draw last segment') + + # # if flip_cache: + # # # # XXX: seems to be needed to avoid artifacts (see above). + # # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + + # # trigger redraw of path + # # do update before reverting to cache mode + # self.update() + # profiler('.update()') def draw_last( self, diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d7ebe4e6d..bc93f6484 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,7 @@ ''' from __future__ import annotations -from functools import partial +# from functools import partial from typing import ( Optional, Callable, @@ -54,6 +54,7 @@ gen_ohlc_qpath, ohlc_to_line, to_step_format, + xy_downsample, ) from ._ohlc import ( BarItems, @@ -152,18 +153,18 @@ def render_baritems( last_read=read, ) - ds_curve_r = Renderer( - flow=self, + # ds_curve_r = Renderer( + # flow=self, - # just swap in the flat view - # data_t=lambda array: self.gy.array, - last_read=read, - draw_path=partial( - rowarr_to_path, - x_basis=None, - ), + # # just swap in the flat view + # # data_t=lambda array: self.gy.array, + # last_read=read, + # draw_path=partial( + # rowarr_to_path, + # x_basis=None, + # ), - ) + # ) curve = FastAppendCurve( name='OHLC', color=graphics._color, @@ -173,12 +174,14 @@ def render_baritems( # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. - self._render_table[0] = ( - ds_curve_r, - curve, - ) + self._render_table[0] = curve + # ( + # # ds_curve_r, + # curve, + # ) - dsc_r, curve = self._render_table[0] + curve = self._render_table[0] + # dsc_r, curve = self._render_table[0] # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to @@ -276,19 +279,20 @@ def render_baritems( profiler('flattened ustruct in-view OHLC data') # pass into curve graphics processing - curve.update_from_array( - x, - y, - x_iv=x_iv, - y_iv=y_iv, - view_range=(ivl, ivr), # hack - profiler=profiler, - # should_redraw=False, - - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - **kwargs, - ) + # curve.update_from_array( + # x, + # y, + # x_iv=x_iv, + # y_iv=y_iv, + # view_range=(ivl, ivr), # hack + # profiler=profiler, + # # should_redraw=False, + + # # NOTE: already passed through by display loop? + # # do_append=uppx < 16, + # **kwargs, + # ) + # curve.draw_last(x, y) curve.show() profiler('updated ds curve') @@ -349,6 +353,130 @@ def render_baritems( # ) # graphics.draw_last(last) + if should_line: + return ( + curve, + x, + y, + x_iv, + y_iv, + ) + + +def update_step_data( + flow: Flow, + shm: ShmArray, + ivl: int, + ivr: int, + array_key: str, + iflat_first: int, + iflat: int, + profiler: pg.debug.Profiler, + +) -> tuple: + + self = flow + ( + # iflat_first, + # iflat, + ishm_last, + ishm_first, + ) = ( + # self._iflat_first, + # self._iflat_last, + shm._last.value, + shm._first.value + ) + il = max(iflat - 1, 0) + profiler('read step mode incr update indices') + + # check for shm prepend updates since last read. + if iflat_first != ishm_first: + + print(f'prepend {array_key}') + + # i_prepend = self.shm._array['index'][ + # ishm_first:iflat_first] + y_prepend = self.shm._array[array_key][ + ishm_first:iflat_first + ] + + y2_prepend = np.broadcast_to( + y_prepend[:, None], (y_prepend.size, 2), + ) + + # write newly prepended data to flattened copy + self.gy[ishm_first:iflat_first] = y2_prepend + self._iflat_first = ishm_first + profiler('prepended step mode history') + + append_diff = ishm_last - iflat + if append_diff: + + # slice up to the last datum since last index/append update + # new_x = self.shm._array[il:ishm_last]['index'] + new_y = self.shm._array[il:ishm_last][array_key] + + new_y2 = np.broadcast_to( + new_y[:, None], (new_y.size, 2), + ) + self.gy[il:ishm_last] = new_y2 + profiler('updated step curve data') + + # print( + # f'append size: {append_diff}\n' + # f'new_x: {new_x}\n' + # f'new_y: {new_y}\n' + # f'new_y2: {new_y2}\n' + # f'new gy: {gy}\n' + # ) + + # update local last-index tracking + self._iflat_last = ishm_last + + # slice out up-to-last step contents + x_step = self.gx[ishm_first:ishm_last+2] + # shape to 1d + x = x_step.reshape(-1) + profiler('sliced step x') + + y_step = self.gy[ishm_first:ishm_last+2] + lasts = self.shm.array[['index', array_key]] + last = lasts[array_key][-1] + y_step[-1] = last + # shape to 1d + y = y_step.reshape(-1) + + # s = 6 + # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') + + profiler('sliced step y') + + # do all the same for only in-view data + ys_iv = y_step[ivl:ivr+1] + xs_iv = x_step[ivl:ivr+1] + y_iv = ys_iv.reshape(ys_iv.size) + x_iv = xs_iv.reshape(xs_iv.size) + # print( + # f'ys_iv : {ys_iv[-s:]}\n' + # f'y_iv: {y_iv[-s:]}\n' + # f'xs_iv: {xs_iv[-s:]}\n' + # f'x_iv: {x_iv[-s:]}\n' + # ) + profiler('sliced in view step data') + + # legacy full-recompute-everytime method + # x, y = ohlc_flatten(array) + # x_iv, y_iv = ohlc_flatten(in_view) + # profiler('flattened OHLC data') + return ( + x, + y, + x_iv, + y_iv, + append_diff, + ) + class Flow(msgspec.Struct): # , frozen=True): ''' @@ -368,11 +496,19 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop + + # pre-graphics formatted data gy: Optional[ShmArray] = None gx: Optional[np.ndarray] = None + # pre-graphics update indices _iflat_last: int = 0 _iflat_first: int = 0 + # view-range incremental state + _vr: Optional[tuple] = None + _avr: Optional[tuple] = None + + # downsampling state _last_uppx: float = 0 _in_ds: bool = False @@ -495,7 +631,11 @@ def datums_range(self) -> tuple[ start, l, lbar, rbar, r, end, ) - def read(self) -> tuple[ + def read( + self, + array_field: Optional[str] = None, + + ) -> tuple[ int, int, np.ndarray, int, int, np.ndarray, ]: @@ -513,6 +653,9 @@ def read(self) -> tuple[ lbar_i = max(l, ifirst) - ifirst rbar_i = min(r, ilast) - ifirst + if array_field: + array = array[array_field] + # TODO: we could do it this way as well no? # to_draw = array[lbar - ifirst:(rbar - ifirst) + 1] in_view = array[lbar_i: rbar_i + 1] @@ -532,6 +675,7 @@ def update_graphics( array_key: Optional[str] = None, profiler: Optional[pg.debug.Profiler] = None, + do_append: bool = True, **kwargs, @@ -557,15 +701,20 @@ def update_graphics( ) = self.read() profiler('read src shm data') + graphics = self.graphics + if ( not in_view.size or not render ): - return self.graphics + return graphics - graphics = self.graphics + out: Optional[tuple] = None if isinstance(graphics, BarItems): - render_baritems( + # XXX: special case where we change out graphics + # to a line after a certain uppx threshold. + # render_baritems( + out = render_baritems( self, graphics, read, @@ -573,14 +722,74 @@ def update_graphics( **kwargs, ) + if out is None: + return graphics + + # return graphics + + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + # draw_path=gen_ohlc_qpath, + last_read=read, + ) + + # ``FastAppendCurve`` case: + array_key = array_key or self.name + + new_sample_rate = False + should_ds = self._in_ds + showing_src_data = self._in_ds + + # draw_last: bool = True + slice_to_head: int = -1 + should_redraw: bool = False + + shm = self.shm + + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + view_range = (ivl, ivr) if use_vr else None + + if out is not None: + # hack to handle ds curve from bars above + ( + graphics, # curve + x, + y, + x_iv, + y_iv, + ) = out + else: - # ``FastAppendCurve`` case: - array_key = array_key or self.name - uppx = graphics.x_uppx() - profiler(f'read uppx {uppx}') + # full input data + x = array['index'] + y = array[array_key] + + # inview data + x_iv = in_view['index'] + y_iv = in_view[array_key] + + # downsampling incremental state checking + uppx = graphics.x_uppx() + # px_width = graphics.px_width() + uppx_diff = (uppx - self._last_uppx) + profiler(f'diffed uppx {uppx}') - if graphics._step_mode and self.gy is None: - shm = self.shm + x_last = x[-1] + y_last = y[-1] + + slice_to_head = -1 + + profiler('sliced input arrays') + + if graphics._step_mode: + slice_to_head = -2 + + if self.gy is None: ( self._iflat_first, self.gx, @@ -591,177 +800,324 @@ def update_graphics( ) profiler('generated step mode data') - if graphics._step_mode: - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value - ) + ( + x, + y, + x_iv, + y_iv, + append_diff, - il = max(iflat - 1, 0) - profiler('read step mode incr update indices') - - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - - print(f'prepend {array_key}') - - # i_prepend = self.shm._array['index'][ - # ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ - ishm_first:iflat_first - ] - - y2_prepend = np.broadcast_to( - y_prepend[:, None], (y_prepend.size, 2), - ) - - # write newly prepended data to flattened copy - self.gy[ishm_first:iflat_first] = y2_prepend - self._iflat_first = ishm_first - profiler('prepended step mode history') - - append_diff = ishm_last - iflat - if append_diff: - - # slice up to the last datum since last index/append update - # new_x = self.shm._array[il:ishm_last]['index'] - new_y = self.shm._array[il:ishm_last][array_key] - - new_y2 = np.broadcast_to( - new_y[:, None], (new_y.size, 2), - ) - self.gy[il:ishm_last] = new_y2 - profiler('updated step curve data') - - # print( - # f'append size: {append_diff}\n' - # f'new_x: {new_x}\n' - # f'new_y: {new_y}\n' - # f'new_y2: {new_y2}\n' - # f'new gy: {gy}\n' - # ) - - # update local last-index tracking - self._iflat_last = ishm_last - - # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last+2] - # shape to 1d - x = x_step.reshape(-1) - profiler('sliced step x') - - y_step = self.gy[ishm_first:ishm_last+2] - lasts = self.shm.array[['index', array_key]] - last = lasts[array_key][-1] - y_step[-1] = last - # shape to 1d - y = y_step.reshape(-1) - - # s = 6 - # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - - profiler('sliced step y') - - # do all the same for only in-view data - ys_iv = y_step[ivl:ivr+1] - xs_iv = x_step[ivl:ivr+1] - y_iv = ys_iv.reshape(ys_iv.size) - x_iv = xs_iv.reshape(xs_iv.size) - # print( - # f'ys_iv : {ys_iv[-s:]}\n' - # f'y_iv: {y_iv[-s:]}\n' - # f'xs_iv: {xs_iv[-s:]}\n' - # f'x_iv: {x_iv[-s:]}\n' - # ) - profiler('sliced in view step data') - - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - - x_last = array['index'][-1] - y_last = array[array_key][-1] - graphics._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, - ) - graphics._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last, + ) = update_step_data( + self, + shm, + ivl, + ivr, + array_key, + self._iflat_first, + self._iflat_last, + profiler, + ) + + graphics._last_line = QLineF( + x_last - 0.5, 0, + x_last + 0.5, 0, + ) + graphics._last_step_rect = QRectF( + x_last - 0.5, 0, + x_last + 0.5, y_last, + ) + + should_redraw = bool(append_diff) + + # graphics.reset_cache() + # print( + # f"path br: {graphics.path.boundingRect()}\n", + # # f"fast path br: {graphics.fast_path.boundingRect()}", + # f"last rect br: {graphics._last_step_rect}\n", + # f"full br: {graphics._br}\n", + # ) + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length, append_length = r.diff(read) + # print((prepend_length, append_length)) + + # old_prepend_length = int(istart - x[0]) + # old_append_length = int(x[-1] - istop) + + # MAIN RENDER LOGIC: + # - determine in view data and redraw on range change + # - determine downsampling ops if needed + # - (incrementally) update ``QPainterPath`` + + if ( + view_range + # and not self._in_ds + # and not prepend_length > 0 + ): + # print(f'{self._name} vr: {view_range}') + + # by default we only pull data up to the last (current) index + x_out = x_iv[:slice_to_head] + y_out = y_iv[:slice_to_head] + profiler(f'view range slice {view_range}') + + vl, vr = view_range + + zoom_or_append = False + last_vr = self._vr + last_ivr = self._avr + + # incremental in-view data update. + if last_vr: + # relative slice indices + lvl, lvr = last_vr + # abs slice indices + al, ar = last_ivr + + # append_length = int(x[-1] - istop) + # append_length = int(x_iv[-1] - ar) + + # left_change = abs(x_iv[0] - al) >= 1 + # right_change = abs(x_iv[-1] - ar) >= 1 + + if ( + # likely a zoom view change + (vr - lvr) > 2 or vl < lvl + # append / prepend update + # we had an append update where the view range + # didn't change but the data-viewed (shifted) + # underneath, so we need to redraw. + # or left_change and right_change and last_vr == view_range + + # not (left_change and right_change) and ivr + # ( + # or abs(x_iv[ivr] - livr) > 1 + ): + zoom_or_append = True + + # if last_ivr: + # liivl, liivr = last_ivr + + if ( + view_range != last_vr + and ( + append_length > 1 + or zoom_or_append ) - # graphics.update() + ): + should_redraw = True + # print("REDRAWING BRUH") - graphics.update_from_array( - x=x, - y=y, + self._vr = view_range + self._avr = x_iv[0], x_iv[slice_to_head] - x_iv=x_iv, - y_iv=y_iv, + if prepend_length > 0: + should_redraw = True - view_range=(ivl, ivr) if use_vr else None, + # check for downsampling conditions + if ( + # std m4 downsample conditions + # px_width + # and abs(uppx_diff) >= 1 + abs(uppx_diff) >= 1 + ): + log.info( + f'{array_key} sampler change: {self._last_uppx} -> {uppx}' + ) + self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False + should_redraw = True + should_ds = True + + elif ( + uppx <= 2 + and self._in_ds + ): + # we should de-downsample back to our original + # source data so we clear our path data in prep + # to generate a new one from original source data. + should_redraw = True + new_sample_rate = True + should_ds = False + showing_src_data = True + + # no_path_yet = self.path is None + fast_path = graphics.fast_path + if ( + graphics.path is None + or should_redraw + or new_sample_rate + or prepend_length > 0 + ): + if should_redraw: + if graphics.path: + graphics.path.clear() + profiler('cleared paths due to `should_redraw=True`') + + if graphics.fast_path: + graphics.fast_path.clear() - draw_last=False, - slice_to_head=-2, + profiler('cleared paths due to `should_redraw` set') - should_redraw=bool(append_diff), + if new_sample_rate and showing_src_data: + # if self._in_ds: + log.info(f'DEDOWN -> {self.name}') - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - profiler=profiler, + self._in_ds = False - **kwargs + # elif should_ds and uppx and px_width > 1: + elif should_ds and uppx > 1: + + x_out, y_out = xy_downsample( + x_out, + y_out, + uppx, + # px_width, ) - profiler('updated step mode curve') - # graphics.reset_cache() - # print( - # f"path br: {graphics.path.boundingRect()}\n", - # # f"fast path br: {graphics.fast_path.boundingRect()}", - # f"last rect br: {graphics._last_step_rect}\n", - # f"full br: {graphics._br}\n", - # ) + profiler(f'FULL PATH downsample redraw={should_ds}') + self._in_ds = True + + graphics.path = pg.functions.arrayToQPath( + x_out, + y_out, + connect='all', + finiteCheck=False, + path=graphics.path, + ) + graphics.prepareGeometryChange() + profiler( + 'generated fresh path. ' + f'(should_redraw: {should_redraw} ' + f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' + ) + # profiler(f'DRAW PATH IN VIEW -> {self.name}') + + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + # if no_path_yet: + # graphics.path.reserve(int(500e3)) + + # TODO: get this piecewise prepend working - right now it's + # giving heck on vwap... + # elif prepend_length: + # breakpoint() + + # prepend_path = pg.functions.arrayToQPath( + # x[0:prepend_length], + # y[0:prepend_length], + # connect='all' + # ) + + # # swap prepend path in "front" + # old_path = graphics.path + # graphics.path = prepend_path + # # graphics.path.moveTo(new_x[0], new_y[0]) + # graphics.path.connectPath(old_path) + + elif ( + append_length > 0 + and do_append + and not should_redraw + # and not view_range + ): + print(f'{self.name} append len: {append_length}') + new_x = x[-append_length - 2:slice_to_head] + new_y = y[-append_length - 2:slice_to_head] + profiler('sliced append path') + profiler( + f'diffed array input, append_length={append_length}' + ) + + # if should_ds: + # new_x, new_y = xy_downsample( + # new_x, + # new_y, + # px_width, + # uppx, + # ) + # profiler(f'fast path downsample redraw={should_ds}') + + append_path = pg.functions.arrayToQPath( + new_x, + new_y, + connect='all', + finiteCheck=False, + path=graphics.fast_path, + ) + profiler('generated append qpath') + + if graphics.use_fpath: + print("USING FPATH") + # an attempt at trying to make append-updates faster.. + if fast_path is None: + graphics.fast_path = append_path + # self.fast_path.reserve(int(6e3)) + else: + fast_path.connectPath(append_path) + size = fast_path.capacity() + profiler(f'connected fast path w size: {size}') + + # print(f"append_path br: {append_path.boundingRect()}") + # graphics.path.moveTo(new_x[0], new_y[0]) + # path.connectPath(append_path) + + # XXX: lol this causes a hang.. + # graphics.path = graphics.path.simplified() else: - x = array['index'] - y = array[array_key] - x_iv = in_view['index'] - y_iv = in_view[array_key] - profiler('sliced input arrays') + size = graphics.path.capacity() + profiler(f'connected history path w size: {size}') + graphics.path.connectPath(append_path) - # graphics.draw_last(x, y) + # graphics.update_from_array( + # x=x, + # y=y, - graphics.update_from_array( - x=x, - y=y, + # x_iv=x_iv, + # y_iv=y_iv, - x_iv=x_iv, - y_iv=y_iv, + # view_range=(ivl, ivr) if use_vr else None, - view_range=(ivl, ivr) if use_vr else None, + # # NOTE: already passed through by display loop. + # # do_append=uppx < 16, + # do_append=do_append, - # NOTE: already passed through by display loop? - # do_append=uppx < 16, - profiler=profiler, - **kwargs - ) - profiler('`graphics.update_from_array()` complete') + # slice_to_head=slice_to_head, + # should_redraw=should_redraw, + # profiler=profiler, + # **kwargs + # ) + + graphics.draw_last(x, y) + profiler('draw last segment') + graphics.update() + profiler('.update()') + + profiler('`graphics.update_from_array()` complete') return graphics class Renderer(msgspec.Struct): flow: Flow + # last array view read + last_read: Optional[tuple] = None # called to render path graphics - draw_path: Callable[np.ndarray, QPainterPath] + draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None + + # output graphics rendering, the main object + # processed in ``QGraphicsObject.paint()`` + path: Optional[QPainterPath] = None # called on input data but before any graphics format # conversions or processing. @@ -778,25 +1134,66 @@ class Renderer(msgspec.Struct): prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - # last array view read - last_read: Optional[np.ndarray] = None + # incremental update state(s) + # _in_ds: bool = False + # _last_uppx: float = 0 + _last_vr: Optional[tuple[float, float]] = None + _last_ivr: Optional[tuple[float, float]] = None - # output graphics rendering, the main object - # processed in ``QGraphicsObject.paint()`` - path: Optional[QPainterPath] = None + def diff( + self, + new_read: tuple[np.ndarray], - # def diff( - # self, - # latest_read: tuple[np.ndarray], + ) -> tuple[np.ndarray]: - # ) -> tuple[np.ndarray]: - # # blah blah blah - # # do diffing for prepend, append and last entry - # return ( - # to_prepend - # to_append - # last, - # ) + ( + last_xfirst, + last_xlast, + last_array, + last_ivl, last_ivr, + last_in_view, + ) = self.last_read + + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read + ( + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length = int(last_xfirst - xfirst) + append_length = int(xlast - last_xlast) + + # TODO: eventually maybe we can implement some kind of + # transform on the ``QPainterPath`` that will more or less + # detect the diff in "elements" terms? + # update state + self.last_read = new_read + + # blah blah blah + # do diffing for prepend, append and last entry + return ( + prepend_length, + append_length, + # last, + ) + + def draw_path( + self, + should_redraw: bool = False, + ) -> QPainterPath: + + if should_redraw: + if self.path: + self.path.clear() + # profiler('cleared paths due to `should_redraw=True`') def render( self, @@ -819,11 +1216,30 @@ def render( - blah blah blah (from notes) ''' - # do full source data render to path + # get graphics info + + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read ( - xfirst, xlast, array, - ivl, ivr, in_view, - ) = self.last_read + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + ( + prepend_length, + append_length, + ) = self.diff(new_read) + + # do full source data render to path + + # x = array['index'] + # y = array#[array_key] + # x_iv = in_view['index'] + # y_iv = in_view#[array_key] if only_in_view: array = in_view @@ -832,7 +1248,10 @@ def render( # xfirst, xlast, array, ivl, ivr, in_view # ) = new_read - if self.path is None or only_in_view: + if ( + self.path is None + or only_in_view + ): # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view From 1dab77ca0b9debc976411ff253a23c58048ce6e1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:46:09 -0400 Subject: [PATCH 063/113] Rect wont show on step curves unless we avoid `.draw_last()` --- piker/ui/_flows.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bc93f6484..b0f1bb21a 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -739,21 +739,16 @@ def update_graphics( # ``FastAppendCurve`` case: array_key = array_key or self.name + shm = self.shm + # update config new_sample_rate = False should_ds = self._in_ds showing_src_data = self._in_ds - - # draw_last: bool = True + draw_last: bool = True slice_to_head: int = -1 should_redraw: bool = False - shm = self.shm - - # if a view range is passed, plan to draw the - # source ouput that's "in view" of the chart. - view_range = (ivl, ivr) if use_vr else None - if out is not None: # hack to handle ds curve from bars above ( @@ -828,6 +823,7 @@ def update_graphics( ) should_redraw = bool(append_diff) + draw_last = False # graphics.reset_cache() # print( @@ -841,10 +837,6 @@ def update_graphics( # the input data and the last indexes we have on record from the # last time we updated the curve index. prepend_length, append_length = r.diff(read) - # print((prepend_length, append_length)) - - # old_prepend_length = int(istart - x[0]) - # old_append_length = int(x[-1] - istop) # MAIN RENDER LOGIC: # - determine in view data and redraw on range change @@ -852,10 +844,14 @@ def update_graphics( # - (incrementally) update ``QPainterPath`` if ( - view_range + use_vr # and not self._in_ds # and not prepend_length > 0 ): + + # if a view range is passed, plan to draw the + # source ouput that's "in view" of the chart. + view_range = (ivl, ivr) # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index @@ -1096,8 +1092,9 @@ def update_graphics( # **kwargs # ) - graphics.draw_last(x, y) - profiler('draw last segment') + if draw_last: + graphics.draw_last(x, y) + profiler('draw last segment') graphics.update() profiler('.update()') From b5b9ecf4b1f3a19734e5654c614f26c92b3a3b29 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 08:53:35 -0400 Subject: [PATCH 064/113] Treat paths like input/output vars --- piker/ui/_flows.py | 48 ++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index b0f1bb21a..a9c3f3d5d 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -843,6 +843,9 @@ def update_graphics( # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` + path = graphics.path + fast_path = graphics.fast_path + if ( use_vr # and not self._in_ds @@ -940,21 +943,19 @@ def update_graphics( should_ds = False showing_src_data = True - # no_path_yet = self.path is None - fast_path = graphics.fast_path if ( - graphics.path is None + path is None or should_redraw or new_sample_rate or prepend_length > 0 ): if should_redraw: - if graphics.path: - graphics.path.clear() + if path: + path.clear() profiler('cleared paths due to `should_redraw=True`') - if graphics.fast_path: - graphics.fast_path.clear() + if fast_path: + fast_path.clear() profiler('cleared paths due to `should_redraw` set') @@ -976,12 +977,12 @@ def update_graphics( profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True - graphics.path = pg.functions.arrayToQPath( + path = pg.functions.arrayToQPath( x_out, y_out, connect='all', finiteCheck=False, - path=graphics.path, + path=path, ) graphics.prepareGeometryChange() profiler( @@ -1047,7 +1048,7 @@ def update_graphics( new_y, connect='all', finiteCheck=False, - path=graphics.fast_path, + path=fast_path, ) profiler('generated append qpath') @@ -1055,8 +1056,8 @@ def update_graphics( print("USING FPATH") # an attempt at trying to make append-updates faster.. if fast_path is None: - graphics.fast_path = append_path - # self.fast_path.reserve(int(6e3)) + fast_path = append_path + # fast_path.reserve(int(6e3)) else: fast_path.connectPath(append_path) size = fast_path.capacity() @@ -1073,25 +1074,6 @@ def update_graphics( profiler(f'connected history path w size: {size}') graphics.path.connectPath(append_path) - # graphics.update_from_array( - # x=x, - # y=y, - - # x_iv=x_iv, - # y_iv=y_iv, - - # view_range=(ivl, ivr) if use_vr else None, - - # # NOTE: already passed through by display loop. - # # do_append=uppx < 16, - # do_append=do_append, - - # slice_to_head=slice_to_head, - # should_redraw=should_redraw, - # profiler=profiler, - # **kwargs - # ) - if draw_last: graphics.draw_last(x, y) profiler('draw last segment') @@ -1099,6 +1081,10 @@ def update_graphics( graphics.update() profiler('.update()') + # assign output paths to graphicis obj + graphics.path = path + graphics.fast_path = fast_path + profiler('`graphics.update_from_array()` complete') return graphics From b3ae562e4f43d4b72c9e9fa270dfdef3fc2a1781 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 09:08:08 -0400 Subject: [PATCH 065/113] Fully drop `.update_from_array()` --- piker/ui/_curve.py | 341 +-------------------------------------------- 1 file changed, 7 insertions(+), 334 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 9e1f684ab..965d682cd 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -84,8 +84,8 @@ def __init__( # brutaaalll, see comments within.. self.yData = None self.xData = None - self._vr: Optional[tuple] = None - self._avr: Optional[tuple] = None + # self._vr: Optional[tuple] = None + # self._avr: Optional[tuple] = None self._last_cap: int = 0 self._name = name @@ -99,12 +99,12 @@ def __init__( super().__init__(*args, **kwargs) # self._xrange: tuple[int, int] = self.dataBounds(ax=0) - self._xrange: Optional[tuple[int, int]] = None + # self._xrange: Optional[tuple[int, int]] = None # self._x_iv_range = None # self._last_draw = time.time() - self._in_ds: bool = False - self._last_uppx: float = 0 + # self._in_ds: bool = False + # self._last_uppx: float = 0 # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) @@ -161,8 +161,8 @@ def px_width(self) -> float: vr = self.viewRect() l, r = int(vr.left()), int(vr.right()) - if not self._xrange: - return 0 + # if not self._xrange: + # return 0 start, stop = self._xrange lbar = max(l, start) @@ -172,333 +172,6 @@ def px_width(self) -> float: QLineF(lbar, 0, rbar, 0) ).length() - # def update_from_array( - # self, - - # # full array input history - # x: np.ndarray, - # y: np.ndarray, - - # # pre-sliced array data that's "in view" - # x_iv: np.ndarray, - # y_iv: np.ndarray, - - # view_range: Optional[tuple[int, int]] = None, - # profiler: Optional[pg.debug.Profiler] = None, - # draw_last: bool = True, - # slice_to_head: int = -1, - # do_append: bool = True, - # should_redraw: bool = False, - - # ) -> QtGui.QPainterPath: - # ''' - # Update curve from input 2-d data. - - # Compare with a cached "x-range" state and (pre/a)ppend based on - # a length diff. - - # ''' - # profiler = profiler or pg.debug.Profiler( - # msg=f'FastAppendCurve.update_from_array(): `{self._name}`', - # disabled=not pg_profile_enabled(), - # ms_threshold=ms_slower_then, - # ) - # # flip_cache = False - - # if self._xrange: - # istart, istop = self._xrange - # else: - # self._xrange = istart, istop = x[0], x[-1] - - # # compute the length diffs between the first/last index entry in - # # the input data and the last indexes we have on record from the - # # last time we updated the curve index. - # prepend_length = int(istart - x[0]) - # append_length = int(x[-1] - istop) - - # # this is the diff-mode, "data"-rendered index - # # tracking var.. - # self._xrange = x[0], x[-1] - - # # print(f"xrange: {self._xrange}") - - # # XXX: lol brutal, the internals of `CurvePoint` (inherited by - # # our `LineDot`) required ``.getData()`` to work.. - # self.xData = x - # self.yData = y - - # # downsampling incremental state checking - # uppx = self.x_uppx() - # px_width = self.px_width() - # uppx_diff = (uppx - self._last_uppx) - - # new_sample_rate = False - # should_ds = self._in_ds - # showing_src_data = self._in_ds - # # should_redraw = False - - # # by default we only pull data up to the last (current) index - # x_out = x[:slice_to_head] - # y_out = y[:slice_to_head] - - # # if a view range is passed, plan to draw the - # # source ouput that's "in view" of the chart. - # if ( - # view_range - # # and not self._in_ds - # # and not prepend_length > 0 - # ): - # # print(f'{self._name} vr: {view_range}') - - # # by default we only pull data up to the last (current) index - # x_out, y_out = x_iv[:slice_to_head], y_iv[:slice_to_head] - # profiler(f'view range slice {view_range}') - - # vl, vr = view_range - - # # last_ivr = self._x_iv_range - # # ix_iv, iy_iv = self._x_iv_range = (x_iv[0], x_iv[-1]) - - # zoom_or_append = False - # last_vr = self._vr - # last_ivr = self._avr - - # if last_vr: - # # relative slice indices - # lvl, lvr = last_vr - # # abs slice indices - # al, ar = last_ivr - - # # append_length = int(x[-1] - istop) - # # append_length = int(x_iv[-1] - ar) - - # # left_change = abs(x_iv[0] - al) >= 1 - # # right_change = abs(x_iv[-1] - ar) >= 1 - - # if ( - # # likely a zoom view change - # (vr - lvr) > 2 or vl < lvl - # # append / prepend update - # # we had an append update where the view range - # # didn't change but the data-viewed (shifted) - # # underneath, so we need to redraw. - # # or left_change and right_change and last_vr == view_range - - # # not (left_change and right_change) and ivr - # # ( - # # or abs(x_iv[ivr] - livr) > 1 - # ): - # zoom_or_append = True - - # # if last_ivr: - # # liivl, liivr = last_ivr - - # if ( - # view_range != last_vr - # and ( - # append_length > 1 - # or zoom_or_append - # ) - # ): - # should_redraw = True - # # print("REDRAWING BRUH") - - # self._vr = view_range - # self._avr = x_iv[0], x_iv[slice_to_head] - - # # x_last = x_iv[-1] - # # y_last = y_iv[-1] - # # self._last_vr = view_range - - # # self.disable_cache() - # # flip_cache = True - - # if prepend_length > 0: - # should_redraw = True - - # # check for downsampling conditions - # if ( - # # std m4 downsample conditions - # abs(uppx_diff) >= 1 - # ): - # log.info( - # f'{self._name} sampler change: {self._last_uppx} -> {uppx}' - # ) - # self._last_uppx = uppx - # new_sample_rate = True - # showing_src_data = False - # should_redraw = True - # should_ds = True - - # elif ( - # uppx <= 2 - # and self._in_ds - # ): - # # we should de-downsample back to our original - # # source data so we clear our path data in prep - # # to generate a new one from original source data. - # should_redraw = True - # new_sample_rate = True - # should_ds = False - # showing_src_data = True - - # # no_path_yet = self.path is None - # if ( - # self.path is None - # or should_redraw - # or new_sample_rate - # or prepend_length > 0 - # ): - # if should_redraw: - # if self.path: - # self.path.clear() - # profiler('cleared paths due to `should_redraw=True`') - - # if self.fast_path: - # self.fast_path.clear() - - # profiler('cleared paths due to `should_redraw` set') - - # if new_sample_rate and showing_src_data: - # # if self._in_ds: - # log.info(f'DEDOWN -> {self._name}') - - # self._in_ds = False - - # elif should_ds and uppx > 1: - - # x_out, y_out = xy_downsample( - # x_out, - # y_out, - # uppx, - # ) - # profiler(f'FULL PATH downsample redraw={should_ds}') - # self._in_ds = True - - # self.path = pg.functions.arrayToQPath( - # x_out, - # y_out, - # connect='all', - # finiteCheck=False, - # path=self.path, - # ) - # self.prepareGeometryChange() - # profiler( - # 'generated fresh path. ' - # f'(should_redraw: {should_redraw} ' - # f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' - # ) - # # profiler(f'DRAW PATH IN VIEW -> {self._name}') - - # # reserve mem allocs see: - # # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # # XXX: right now this is based on had hoc checks on a - # # hidpi 3840x2160 4k monitor but we should optimize for - # # the target display(s) on the sys. - # # if no_path_yet: - # # self.path.reserve(int(500e3)) - - # # TODO: get this piecewise prepend working - right now it's - # # giving heck on vwap... - # # elif prepend_length: - # # breakpoint() - - # # prepend_path = pg.functions.arrayToQPath( - # # x[0:prepend_length], - # # y[0:prepend_length], - # # connect='all' - # # ) - - # # # swap prepend path in "front" - # # old_path = self.path - # # self.path = prepend_path - # # # self.path.moveTo(new_x[0], new_y[0]) - # # self.path.connectPath(old_path) - - # elif ( - # append_length > 0 - # and do_append - # and not should_redraw - # # and not view_range - # ): - # print(f'{self._name} append len: {append_length}') - # new_x = x[-append_length - 2:slice_to_head] - # new_y = y[-append_length - 2:slice_to_head] - # profiler('sliced append path') - - # profiler( - # f'diffed array input, append_length={append_length}' - # ) - - # # if should_ds: - # # new_x, new_y = xy_downsample( - # # new_x, - # # new_y, - # # uppx, - # # ) - # # profiler(f'fast path downsample redraw={should_ds}') - - # append_path = pg.functions.arrayToQPath( - # new_x, - # new_y, - # connect='all', - # finiteCheck=False, - # path=self.fast_path, - # ) - # profiler('generated append qpath') - - # if self.use_fpath: - # # an attempt at trying to make append-updates faster.. - # if self.fast_path is None: - # self.fast_path = append_path - # # self.fast_path.reserve(int(6e3)) - # else: - # self.fast_path.connectPath(append_path) - # size = self.fast_path.capacity() - # profiler(f'connected fast path w size: {size}') - - # # print(f"append_path br: {append_path.boundingRect()}") - # # self.path.moveTo(new_x[0], new_y[0]) - # # path.connectPath(append_path) - - # # XXX: lol this causes a hang.. - # # self.path = self.path.simplified() - # else: - # size = self.path.capacity() - # profiler(f'connected history path w size: {size}') - # self.path.connectPath(append_path) - - # # other merging ideas: - # # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths - # # path.addPath(append_path) - # # path.closeSubpath() - - # # TODO: try out new work from `pyqtgraph` main which - # # should repair horrid perf: - # # https://github.com/pyqtgraph/pyqtgraph/pull/2032 - # # ok, nope still horrible XD - # # if self._fill: - # # # XXX: super slow set "union" op - # # self.path = self.path.united(append_path).simplified() - - # # self.disable_cache() - # # flip_cache = True - - # # if draw_last: - # # self.draw_last(x, y) - # # profiler('draw last segment') - - # # if flip_cache: - # # # # XXX: seems to be needed to avoid artifacts (see above). - # # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - # # trigger redraw of path - # # do update before reverting to cache mode - # self.update() - # profiler('.update()') - def draw_last( self, x: np.ndarray, From 72e849c6518c77365ab34ef12b01e60f577286a3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 09:08:38 -0400 Subject: [PATCH 066/113] Drop commented cruft from update logic --- piker/ui/_flows.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index a9c3f3d5d..bae4b5a92 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -825,14 +825,6 @@ def update_graphics( should_redraw = bool(append_diff) draw_last = False - # graphics.reset_cache() - # print( - # f"path br: {graphics.path.boundingRect()}\n", - # # f"fast path br: {graphics.fast_path.boundingRect()}", - # f"last rect br: {graphics._last_step_rect}\n", - # f"full br: {graphics._br}\n", - # ) - # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. @@ -848,8 +840,6 @@ def update_graphics( if ( use_vr - # and not self._in_ds - # and not prepend_length > 0 ): # if a view range is passed, plan to draw the @@ -875,9 +865,6 @@ def update_graphics( # abs slice indices al, ar = last_ivr - # append_length = int(x[-1] - istop) - # append_length = int(x_iv[-1] - ar) - # left_change = abs(x_iv[0] - al) >= 1 # right_change = abs(x_iv[-1] - ar) >= 1 @@ -896,9 +883,6 @@ def update_graphics( ): zoom_or_append = True - # if last_ivr: - # liivl, liivr = last_ivr - if ( view_range != last_vr and ( @@ -915,11 +899,8 @@ def update_graphics( if prepend_length > 0: should_redraw = True - # check for downsampling conditions + # check for and set std m4 downsample conditions if ( - # std m4 downsample conditions - # px_width - # and abs(uppx_diff) >= 1 abs(uppx_diff) >= 1 ): log.info( @@ -965,14 +946,12 @@ def update_graphics( self._in_ds = False - # elif should_ds and uppx and px_width > 1: elif should_ds and uppx > 1: x_out, y_out = xy_downsample( x_out, y_out, uppx, - # px_width, ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True @@ -1023,7 +1002,6 @@ def update_graphics( append_length > 0 and do_append and not should_redraw - # and not view_range ): print(f'{self.name} append len: {append_length}') new_x = x[-append_length - 2:slice_to_head] From 876add4fc21dad2def1b23e145f6d838a0622d4f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:16:01 -0400 Subject: [PATCH 067/113] Drop `.update()` call from `.draw_last()` --- piker/ui/_curve.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 965d682cd..fa073d37d 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -207,8 +207,6 @@ def draw_last( x_last, y_last ) - self.update() - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): From 664a208ae51ac1edba35b27d2701c09efc10456b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:17:48 -0400 Subject: [PATCH 068/113] Drop path generation from `gen_ohlc_qpath()` --- piker/ui/_pathops.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index f7eaf2a72..4cb5b86e1 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -33,7 +33,6 @@ ) from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( - # ohlc_flatten, ds_m4, ) @@ -140,43 +139,24 @@ def path_arrays_from_ohlc( def gen_ohlc_qpath( data: np.ndarray, + array_key: str, # we ignore this + start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43, - path: Optional[QtGui.QPainterPath] = None, ) -> QtGui.QPainterPath: + ''' + More or less direct proxy to ``path_arrays_from_ohlc()`` + but with closed in kwargs for line spacing. - path_was_none = path is None - - profiler = pg.debug.Profiler( - msg='gen_qpath ohlc', - disabled=not pg_profile_enabled(), - ms_threshold=ms_slower_then, - ) - + ''' x, y, c = path_arrays_from_ohlc( data, start, bar_gap=w, ) - profiler("generate stream with numba") - - # TODO: numba the internals of this! - path = pg.functions.arrayToQPath( - x, - y, - connect=c, - path=path, - ) - - # avoid mem allocs if possible - if path_was_none: - path.reserve(path.capacity()) - - profiler("generate path with arrayToQPath") - - return path + return x, y, c def ohlc_to_line( From aa0efe15230f3240b9fed0b81a9d52b6ee9949e0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:21:21 -0400 Subject: [PATCH 069/113] Drop `BarItems.draw_from_data()` --- piker/ui/_chart.py | 5 ----- piker/ui/_ohlc.py | 56 ---------------------------------------------- 2 files changed, 61 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 2eba9a24a..31ca06041 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -979,11 +979,6 @@ def draw_ohlc( graphics=graphics, ) - # TODO: i think we can eventually remove this if - # we write the ``Flow.update_graphics()`` method right? - # draw after to allow self.scene() to work... - graphics.draw_from_data(shm.array) - self._add_sticky(name, bg_color='davies') return graphics, data_key diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 88fa62f9e..fb57d6ff4 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -146,62 +146,6 @@ def __init__( self._dsi: tuple[int, int] = 0, 0 self._xs_in_px: float = 0 - def draw_from_data( - self, - ohlc: np.ndarray, - start: int = 0, - - ) -> QtGui.QPainterPath: - ''' - Draw OHLC datum graphics from a ``np.ndarray``. - - This routine is usually only called to draw the initial history. - - ''' - hist, last = ohlc[:-1], ohlc[-1] - self.path = gen_ohlc_qpath(hist, start, self.w) - - # save graphics for later reference and keep track - # of current internal "last index" - # self.start_index = len(ohlc) - index = ohlc['index'] - self._xrange = (index[0], index[-1]) - # self._yrange = ( - # np.nanmax(ohlc['high']), - # np.nanmin(ohlc['low']), - # ) - - # up to last to avoid double draw of last bar - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - x, y = self._ds_line_xy = ohlc_flatten(ohlc) - - # TODO: figuring out the most optimial size for the ideal - # curve-path by, - # - calcing the display's max px width `.screen()` - # - drawing a curve and figuring out it's capacity: - # https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - reserving that cap for each curve-mapped-to-shm with - - # - leveraging clearing when needed to redraw the entire - # curve that does not release mem allocs: - # https://doc.qt.io/qt-5/qpainterpath.html#clear - curve = FastAppendCurve( - name='OHLC', - color=self._color, - ) - curve.hide() - self._pi.addItem(curve) - self._ds_line = curve - - # self._ds_xrange = (index[0], index[-1]) - - # trigger render - # https://doc.qt.io/qt-5/qgraphicsitem.html#update - self.update() - - return self.path - def x_uppx(self) -> int: if self._ds_line: return self._ds_line.x_uppx() From 167ae965665ee132a484eaccc90c49bd8bd743f3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 18 May 2022 15:23:14 -0400 Subject: [PATCH 070/113] Move graphics update logic into `Renderer.render()` Finally this gets us much closer to a generic incremental update system for graphics wherein the input array diffing, pre-graphical format data processing, downsampler activation and incremental update and storage of any of these data flow stages can be managed in one modular sub-system :surfer_boi:. Dirty deatz: - reorg and move all path logic into `Renderer.render()` and have it take in pretty much the same flags as the old `FastAppendCurve.update_from_array()` and instead storing all update state vars (even copies of the downsampler related ones) on the renderer instance: - new state vars: `._last_uppx, ._in_ds, ._vr, ._avr` - `.render()` input bools: `new_sample_rate, should_redraw, should_ds, showing_src_data` - add a hack-around for passing in incremental update data (for now) via a `input_data: tuple` of numpy arrays - a default `uppx: float = 1` - add new render interface attrs: - `.format_xy()` which takes in the source data array and produces out x, y arrays (and maybe a `connect` array) that can be passed to `.draw_path()` (the default for this is just to slice out the index and `array_key: str` columns from the input struct array), - `.draw_path()` which takes in the x, y, connect arrays and generates a `QPainterPath` - `.fast_path`, for "appendable" updates like there was on the fast append curve - move redraw (aka `.clear()` calls) into `.draw_path()` and trigger via `redraw: bool` flag. - our graphics objects no longer set their own `.path` state, it's done by the `Flow.update_graphics()` method using output from `Renderer.render()` (and it's state if necessary) --- piker/ui/_flows.py | 699 ++++++++++++++++++++++++++++----------------- 1 file changed, 439 insertions(+), 260 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index bae4b5a92..c974878df 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -27,6 +27,7 @@ from typing import ( Optional, Callable, + Union, ) import msgspec @@ -144,12 +145,13 @@ def render_baritems( # if no source data renderer exists create one. self = flow r = self._src_r + show_bars: bool = False if not r: + show_bars = True # OHLC bars path renderer r = self._src_r = Renderer( flow=self, - # TODO: rename this to something with ohlc - draw_path=gen_ohlc_qpath, + format_xy=gen_ohlc_qpath, last_read=read, ) @@ -292,20 +294,28 @@ def render_baritems( # # do_append=uppx < 16, # **kwargs, # ) - # curve.draw_last(x, y) + curve.draw_last(x, y) curve.show() profiler('updated ds curve') else: # render incremental or in-view update # and apply ouput (path) to graphics. - path, last = r.render( + path, data = r.render( read, - only_in_view=True, + 'ohlc', + profiler=profiler, + # uppx=1, + use_vr=True, + # graphics=graphics, + # should_redraw=True, # always ) + assert path graphics.path = path - graphics.draw_last(last) + graphics.draw_last(data[-1]) + if show_bars: + graphics.show() # NOTE: on appends we used to have to flip the coords # cache thought it doesn't seem to be required any more? @@ -699,6 +709,7 @@ def update_graphics( xfirst, xlast, array, ivl, ivr, in_view, ) = self.read() + profiler('read src shm data') graphics = self.graphics @@ -709,8 +720,13 @@ def update_graphics( ): return graphics + draw_last: bool = True + slice_to_head: int = -1 + input_data = None + out: Optional[tuple] = None if isinstance(graphics, BarItems): + draw_last = False # XXX: special case where we change out graphics # to a line after a certain uppx threshold. # render_baritems( @@ -741,14 +757,6 @@ def update_graphics( array_key = array_key or self.name shm = self.shm - # update config - new_sample_rate = False - should_ds = self._in_ds - showing_src_data = self._in_ds - draw_last: bool = True - slice_to_head: int = -1 - should_redraw: bool = False - if out is not None: # hack to handle ds curve from bars above ( @@ -758,32 +766,60 @@ def update_graphics( x_iv, y_iv, ) = out + input_data = out[1:] + # breakpoint() - else: - # full input data - x = array['index'] - y = array[array_key] - - # inview data - x_iv = in_view['index'] - y_iv = in_view[array_key] + # ds update config + new_sample_rate: bool = False + should_redraw: bool = False + should_ds: bool = r._in_ds + showing_src_data: bool = not r._in_ds # downsampling incremental state checking + # check for and set std m4 downsample conditions uppx = graphics.x_uppx() - # px_width = graphics.px_width() uppx_diff = (uppx - self._last_uppx) profiler(f'diffed uppx {uppx}') + if ( + uppx > 1 + and abs(uppx_diff) >= 1 + ): + log.info( + f'{array_key} sampler change: {self._last_uppx} -> {uppx}' + ) + self._last_uppx = uppx + new_sample_rate = True + showing_src_data = False + should_redraw = True + should_ds = True - x_last = x[-1] - y_last = y[-1] - - slice_to_head = -1 - - profiler('sliced input arrays') + elif ( + uppx <= 2 + and self._in_ds + ): + # we should de-downsample back to our original + # source data so we clear our path data in prep + # to generate a new one from original source data. + should_redraw = True + new_sample_rate = True + should_ds = False + showing_src_data = True if graphics._step_mode: slice_to_head = -2 + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = array['index'] + y = array[array_key] + x_last = x[-1] + y_last = y[-1] + + # inview data + x_iv = in_view['index'] + y_iv = in_view[array_key] + if self.gy is None: ( self._iflat_first, @@ -824,32 +860,335 @@ def update_graphics( should_redraw = bool(append_diff) draw_last = False + input_data = ( + x, + y, + x_iv, + y_iv, + ) # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the # last time we updated the curve index. - prepend_length, append_length = r.diff(read) + # prepend_length, append_length = r.diff(read) # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path = graphics.path - fast_path = graphics.fast_path + # path = graphics.path + # fast_path = graphics.fast_path + + path, data = r.render( + read, + array_key, + profiler, + uppx=uppx, + input_data=input_data, + # use_vr=True, + + # TODO: better way to detect and pass this? + # if we want to eventually cache renderers for a given uppx + # we should probably use this as a key + state? + should_redraw=should_redraw, + new_sample_rate=new_sample_rate, + should_ds=should_ds, + showing_src_data=showing_src_data, + + slice_to_head=slice_to_head, + do_append=do_append, + graphics=graphics, + ) + # graphics.prepareGeometryChange() + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path + + if draw_last: + x = data['index'] + y = data[array_key] + graphics.draw_last(x, y) + profiler('draw last segment') + + graphics.update() + profiler('.update()') + + profiler('`graphics.update_from_array()` complete') + return graphics + + +def by_index_and_key( + array: np.ndarray, + array_key: str, + +) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, +]: + # full input data + x = array['index'] + y = array[array_key] + + # # inview data + # x_iv = in_view['index'] + # y_iv = in_view[array_key] + + return tuple({ + 'x': x, + 'y': y, + # 'x_iv': x_iv, + # 'y_iv': y_iv, + 'connect': 'all', + }.values()) + + +class Renderer(msgspec.Struct): + + flow: Flow + # last array view read + last_read: Optional[tuple] = None + format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key + + # called to render path graphics + # draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None + + # output graphics rendering, the main object + # processed in ``QGraphicsObject.paint()`` + path: Optional[QPainterPath] = None + fast_path: Optional[QPainterPath] = None + + # called on input data but before any graphics format + # conversions or processing. + data_t: Optional[Callable[ShmArray, np.ndarray]] = None + data_t_shm: Optional[ShmArray] = None + + # called on the final data (transform) output to convert + # to "graphical data form" a format that can be passed to + # the ``.draw()`` implementation. + graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None + graphics_t_shm: Optional[ShmArray] = None + + # path graphics update implementation methods + prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + + # downsampling state + _last_uppx: float = 0 + _in_ds: bool = False + + # incremental update state(s) + _last_vr: Optional[tuple[float, float]] = None + _last_ivr: Optional[tuple[float, float]] = None + + # view-range incremental state + _vr: Optional[tuple] = None + _avr: Optional[tuple] = None + + def diff( + self, + new_read: tuple[np.ndarray], + + ) -> tuple[np.ndarray]: + + ( + last_xfirst, + last_xlast, + last_array, + last_ivl, last_ivr, + last_in_view, + ) = self.last_read + + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read + ( + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + # compute the length diffs between the first/last index entry in + # the input data and the last indexes we have on record from the + # last time we updated the curve index. + prepend_length = int(last_xfirst - xfirst) + append_length = int(xlast - last_xlast) + + # TODO: eventually maybe we can implement some kind of + # transform on the ``QPainterPath`` that will more or less + # detect the diff in "elements" terms? + # update state + self.last_read = new_read + + # blah blah blah + # do diffing for prepend, append and last entry + return ( + prepend_length, + append_length, + # last, + ) + + # def gen_path_data( + # self, + # redraw: bool = False, + # ) -> np.ndarray: + # ... + + def draw_path( + self, + x: np.ndarray, + y: np.ndarray, + connect: Union[str, np.ndarray] = 'all', + path: Optional[QPainterPath] = None, + redraw: bool = False, + + ) -> QPainterPath: + + path_was_none = path is None + + if redraw and path: + path.clear() + + # TODO: avoid this? + if self.fast_path: + self.fast_path.clear() + + # profiler('cleared paths due to `should_redraw=True`') + + path = pg.functions.arrayToQPath( + x, + y, + connect=connect, + finiteCheck=False, + + # reserve mem allocs see: + # - https://doc.qt.io/qt-5/qpainterpath.html#reserve + # - https://doc.qt.io/qt-5/qpainterpath.html#capacity + # - https://doc.qt.io/qt-5/qpainterpath.html#clear + # XXX: right now this is based on had hoc checks on a + # hidpi 3840x2160 4k monitor but we should optimize for + # the target display(s) on the sys. + # if no_path_yet: + # graphics.path.reserve(int(500e3)) + path=path, # path re-use / reserving + ) + + # avoid mem allocs if possible + if path_was_none: + path.reserve(path.capacity()) + + return path + + def render( + self, + + new_read, + array_key: str, + profiler: pg.debug.Profiler, + uppx: float = 1, + + input_data: Optional[tuple[np.ndarray]] = None, + + # redraw and ds flags + should_redraw: bool = True, + new_sample_rate: bool = False, + should_ds: bool = False, + showing_src_data: bool = True, + + do_append: bool = True, + slice_to_head: int = -1, + use_fpath: bool = True, + + # only render datums "in view" of the ``ChartView`` + use_vr: bool = True, + graphics: Optional[pg.GraphicObject] = None, + + ) -> list[QPainterPath]: + ''' + Render the current graphics path(s) + + There are (at least) 3 stages from source data to graphics data: + - a data transform (which can be stored in additional shm) + - a graphics transform which converts discrete basis data to + a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``, + ``step_path_arrays_from_1d()``, etc.) + + - blah blah blah (from notes) + + ''' + # TODO: can the renderer just call ``Flow.read()`` directly? + # unpack latest source data read + ( + xfirst, + xlast, + array, + ivl, + ivr, + in_view, + ) = new_read + + if use_vr: + array = in_view + + if input_data: + # allow input data passing for now from alt curve updaters. + ( + x_out, + y_out, + x_iv, + y_iv, + ) = input_data + connect = 'all' + + if use_vr: + x_out = x_iv + y_out = y_iv + + # last = y_out[slice_to_head] + + else: + hist = array[:slice_to_head] + # last = array[slice_to_head] + + ( + x_out, + y_out, + # x_iv, + # y_iv, + connect, + ) = self.format_xy(hist, array_key) + + # print(f'{array_key} len x,y: {(len(x_out), len(y_out))}') +# # full input data +# x = array['index'] +# y = array[array_key] + +# # inview data +# x_iv = in_view['index'] +# y_iv = in_view[array_key] + + profiler('sliced input arrays') + + ( + prepend_length, + append_length, + ) = self.diff(new_read) if ( use_vr ): - # if a view range is passed, plan to draw the # source ouput that's "in view" of the chart. view_range = (ivl, ivr) # print(f'{self._name} vr: {view_range}') # by default we only pull data up to the last (current) index - x_out = x_iv[:slice_to_head] - y_out = y_iv[:slice_to_head] + # x_out = x_iv[:slice_to_head] + # y_out = y_iv[:slice_to_head] + profiler(f'view range slice {view_range}') vl, vr = view_range @@ -894,35 +1233,18 @@ def update_graphics( # print("REDRAWING BRUH") self._vr = view_range - self._avr = x_iv[0], x_iv[slice_to_head] + if len(x_out): + self._avr = x_out[0], x_out[slice_to_head] if prepend_length > 0: should_redraw = True - # check for and set std m4 downsample conditions - if ( - abs(uppx_diff) >= 1 - ): - log.info( - f'{array_key} sampler change: {self._last_uppx} -> {uppx}' - ) - self._last_uppx = uppx - new_sample_rate = True - showing_src_data = False - should_redraw = True - should_ds = True + # # last datums + # x_last = x_out[-1] + # y_last = y_out[-1] - elif ( - uppx <= 2 - and self._in_ds - ): - # we should de-downsample back to our original - # source data so we clear our path data in prep - # to generate a new one from original source data. - should_redraw = True - new_sample_rate = True - should_ds = False - showing_src_data = True + path = self.path + fast_path = self.fast_path if ( path is None @@ -930,19 +1252,19 @@ def update_graphics( or new_sample_rate or prepend_length > 0 ): - if should_redraw: - if path: - path.clear() - profiler('cleared paths due to `should_redraw=True`') + # if should_redraw: + # if path: + # path.clear() + # profiler('cleared paths due to `should_redraw=True`') - if fast_path: - fast_path.clear() + # if fast_path: + # fast_path.clear() - profiler('cleared paths due to `should_redraw` set') + # profiler('cleared paths due to `should_redraw` set') if new_sample_rate and showing_src_data: # if self._in_ds: - log.info(f'DEDOWN -> {self.name}') + log.info(f'DEDOWN -> {array_key}') self._in_ds = False @@ -955,15 +1277,26 @@ def update_graphics( ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True + # else: + # print(f"NOT DOWNSAMPLING {array_key}") - path = pg.functions.arrayToQPath( - x_out, - y_out, - connect='all', - finiteCheck=False, + path = self.draw_path( + x=x_out, + y=y_out, + connect=connect, path=path, + redraw=True, ) - graphics.prepareGeometryChange() + # path = pg.functions.arrayToQPath( + # x_out, + # y_out, + # connect='all', + # finiteCheck=False, + # path=path, + # ) + if graphics: + graphics.prepareGeometryChange() + profiler( 'generated fresh path. ' f'(should_redraw: {should_redraw} ' @@ -971,16 +1304,6 @@ def update_graphics( ) # profiler(f'DRAW PATH IN VIEW -> {self.name}') - # reserve mem allocs see: - # - https://doc.qt.io/qt-5/qpainterpath.html#reserve - # - https://doc.qt.io/qt-5/qpainterpath.html#capacity - # - https://doc.qt.io/qt-5/qpainterpath.html#clear - # XXX: right now this is based on had hoc checks on a - # hidpi 3840x2160 4k monitor but we should optimize for - # the target display(s) on the sys. - # if no_path_yet: - # graphics.path.reserve(int(500e3)) - # TODO: get this piecewise prepend working - right now it's # giving heck on vwap... # elif prepend_length: @@ -1003,9 +1326,10 @@ def update_graphics( and do_append and not should_redraw ): - print(f'{self.name} append len: {append_length}') - new_x = x[-append_length - 2:slice_to_head] - new_y = y[-append_length - 2:slice_to_head] + # print(f'{self.name} append len: {append_length}') + print(f'{array_key} append len: {append_length}') + new_x = x_out[-append_length - 2:] # slice_to_head] + new_y = y_out[-append_length - 2:] # slice_to_head] profiler('sliced append path') profiler( @@ -1016,21 +1340,26 @@ def update_graphics( # new_x, new_y = xy_downsample( # new_x, # new_y, - # px_width, # uppx, # ) # profiler(f'fast path downsample redraw={should_ds}') - append_path = pg.functions.arrayToQPath( - new_x, - new_y, - connect='all', - finiteCheck=False, - path=fast_path, + append_path = self.draw_path( + x=new_x, + y=new_y, + connect=connect, + # path=fast_path, ) + + # append_path = pg.functions.arrayToQPath( + # connect='all', + # finiteCheck=False, + # path=fast_path, + # ) profiler('generated append qpath') - if graphics.use_fpath: + # if graphics.use_fpath: + if use_fpath: print("USING FPATH") # an attempt at trying to make append-updates faster.. if fast_path is None: @@ -1048,171 +1377,21 @@ def update_graphics( # XXX: lol this causes a hang.. # graphics.path = graphics.path.simplified() else: - size = graphics.path.capacity() + size = path.capacity() profiler(f'connected history path w size: {size}') - graphics.path.connectPath(append_path) - - if draw_last: - graphics.draw_last(x, y) - profiler('draw last segment') - - graphics.update() - profiler('.update()') - - # assign output paths to graphicis obj - graphics.path = path - graphics.fast_path = fast_path - - profiler('`graphics.update_from_array()` complete') - return graphics - - -class Renderer(msgspec.Struct): - - flow: Flow - # last array view read - last_read: Optional[tuple] = None - - # called to render path graphics - draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None - - # output graphics rendering, the main object - # processed in ``QGraphicsObject.paint()`` - path: Optional[QPainterPath] = None - - # called on input data but before any graphics format - # conversions or processing. - data_t: Optional[Callable[ShmArray, np.ndarray]] = None - data_t_shm: Optional[ShmArray] = None - - # called on the final data (transform) output to convert - # to "graphical data form" a format that can be passed to - # the ``.draw()`` implementation. - graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None - graphics_t_shm: Optional[ShmArray] = None - - # path graphics update implementation methods - prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - - # incremental update state(s) - # _in_ds: bool = False - # _last_uppx: float = 0 - _last_vr: Optional[tuple[float, float]] = None - _last_ivr: Optional[tuple[float, float]] = None - - def diff( - self, - new_read: tuple[np.ndarray], - - ) -> tuple[np.ndarray]: - - ( - last_xfirst, - last_xlast, - last_array, - last_ivl, last_ivr, - last_in_view, - ) = self.last_read - - # TODO: can the renderer just call ``Flow.read()`` directly? - # unpack latest source data read - ( - xfirst, - xlast, - array, - ivl, - ivr, - in_view, - ) = new_read - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - prepend_length = int(last_xfirst - xfirst) - append_length = int(xlast - last_xlast) - - # TODO: eventually maybe we can implement some kind of - # transform on the ``QPainterPath`` that will more or less - # detect the diff in "elements" terms? - # update state - self.last_read = new_read - - # blah blah blah - # do diffing for prepend, append and last entry - return ( - prepend_length, - append_length, - # last, - ) - - def draw_path( - self, - should_redraw: bool = False, - ) -> QPainterPath: - - if should_redraw: - if self.path: - self.path.clear() - # profiler('cleared paths due to `should_redraw=True`') - - def render( - self, - - new_read, - - # only render datums "in view" of the ``ChartView`` - only_in_view: bool = False, - - ) -> list[QPainterPath]: - ''' - Render the current graphics path(s) - - There are (at least) 3 stages from source data to graphics data: - - a data transform (which can be stored in additional shm) - - a graphics transform which converts discrete basis data to - a `float`-basis view-coords graphics basis. (eg. ``ohlc_flatten()``, - ``step_path_arrays_from_1d()``, etc.) + path.connectPath(append_path) - - blah blah blah (from notes) - - ''' - # get graphics info - - # TODO: can the renderer just call ``Flow.read()`` directly? - # unpack latest source data read - ( - xfirst, - xlast, - array, - ivl, - ivr, - in_view, - ) = new_read - - ( - prepend_length, - append_length, - ) = self.diff(new_read) - - # do full source data render to path - - # x = array['index'] - # y = array#[array_key] - # x_iv = in_view['index'] - # y_iv = in_view#[array_key] - - if only_in_view: - array = in_view + # if use_vr: + # array = in_view # # get latest data from flow shm # self.last_read = ( # xfirst, xlast, array, ivl, ivr, in_view # ) = new_read - if ( - self.path is None - or only_in_view - ): + # if ( + # self.path is None + # or use_vr + # ): # redraw the entire source data if we have either of: # - no prior path graphic rendered or, # - we always intend to re-render the data only in view @@ -1220,8 +1399,8 @@ def render( # data transform: convert source data to a format # expected to be incrementally updates and later rendered # to a more graphics native format. - if self.data_t: - array = self.data_t(array) + # if self.data_t: + # array = self.data_t(array) # maybe allocate shm for data transform output # if self.data_t_shm is None: @@ -1237,18 +1416,18 @@ def render( # shm.push(array) # self.data_t_shm = shm - elif self.path: - print(f'inremental update not supported yet {self.flow.name}') + # elif self.path: + # print(f'inremental update not supported yet {self.flow.name}') # TODO: do incremental update # prepend, append, last = self.diff(self.flow.read()) # do path generation for each segment # and then push into graphics object. - hist, last = array[:-1], array[-1] - # call path render func on history - self.path = self.draw_path(hist) + # self.path = self.draw_path(hist) + self.path = path + self.fast_path = fast_path self.last_read = new_read - return self.path, last + return self.path, array From 17456d96e0d85a425be38658879d35057865b70a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 19 May 2022 10:23:59 -0400 Subject: [PATCH 071/113] Drop tons of old cruft, move around some commented ideas --- piker/ui/_flows.py | 186 ++++++++++++--------------------------------- 1 file changed, 48 insertions(+), 138 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c974878df..d7c5c2e63 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -510,14 +510,11 @@ class Flow(msgspec.Struct): # , frozen=True): # pre-graphics formatted data gy: Optional[ShmArray] = None gx: Optional[np.ndarray] = None + # pre-graphics update indices _iflat_last: int = 0 _iflat_first: int = 0 - # view-range incremental state - _vr: Optional[tuple] = None - _avr: Optional[tuple] = None - # downsampling state _last_uppx: float = 0 _in_ds: bool = False @@ -551,13 +548,12 @@ def shm(self) -> ShmArray: # private ``._shm`` attr? @shm.setter def shm(self, shm: ShmArray) -> ShmArray: - print(f'{self.name} DO NOT SET SHM THIS WAY!?') self._shm = shm def maxmin( self, - lbar, - rbar, + lbar: int, + rbar: int, ) -> tuple[float, float]: ''' @@ -813,8 +809,6 @@ def update_graphics( # full input data x = array['index'] y = array[array_key] - x_last = x[-1] - y_last = y[-1] # inview data x_iv = in_view['index'] @@ -849,6 +843,8 @@ def update_graphics( profiler, ) + x_last = x[-1] + y_last = y[-1] graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, @@ -898,9 +894,11 @@ def update_graphics( slice_to_head=slice_to_head, do_append=do_append, - graphics=graphics, ) + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). # graphics.prepareGeometryChange() + # assign output paths to graphicis obj graphics.path = r.path graphics.fast_path = r.fast_path @@ -931,15 +929,9 @@ def by_index_and_key( x = array['index'] y = array[array_key] - # # inview data - # x_iv = in_view['index'] - # y_iv = in_view[array_key] - return tuple({ 'x': x, 'y': y, - # 'x_iv': x_iv, - # 'y_iv': y_iv, 'connect': 'all', }.values()) @@ -951,9 +943,6 @@ class Renderer(msgspec.Struct): last_read: Optional[tuple] = None format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key - # called to render path graphics - # draw_path: Optional[Callable[np.ndarray, QPainterPath]] = None - # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None @@ -961,18 +950,18 @@ class Renderer(msgspec.Struct): # called on input data but before any graphics format # conversions or processing. - data_t: Optional[Callable[ShmArray, np.ndarray]] = None - data_t_shm: Optional[ShmArray] = None + format_data: Optional[Callable[ShmArray, np.ndarray]] = None + # XXX: just ideas.. # called on the final data (transform) output to convert # to "graphical data form" a format that can be passed to # the ``.draw()`` implementation. - graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None - graphics_t_shm: Optional[ShmArray] = None + # graphics_t: Optional[Callable[ShmArray, np.ndarray]] = None + # graphics_t_shm: Optional[ShmArray] = None # path graphics update implementation methods - prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None - append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + # prepend_fn: Optional[Callable[QPainterPath, QPainterPath]] = None + # append_fn: Optional[Callable[QPainterPath, QPainterPath]] = None # downsampling state _last_uppx: float = 0 @@ -982,21 +971,20 @@ class Renderer(msgspec.Struct): _last_vr: Optional[tuple[float, float]] = None _last_ivr: Optional[tuple[float, float]] = None - # view-range incremental state - _vr: Optional[tuple] = None - _avr: Optional[tuple] = None - def diff( self, new_read: tuple[np.ndarray], - ) -> tuple[np.ndarray]: - + ) -> tuple[ + np.ndarray, + np.ndarray, + ]: ( last_xfirst, last_xlast, last_array, - last_ivl, last_ivr, + last_ivl, + last_ivr, last_in_view, ) = self.last_read @@ -1028,15 +1016,8 @@ def diff( return ( prepend_length, append_length, - # last, ) - # def gen_path_data( - # self, - # redraw: bool = False, - # ) -> np.ndarray: - # ... - def draw_path( self, x: np.ndarray, @@ -1104,7 +1085,6 @@ def render( # only render datums "in view" of the ``ChartView`` use_vr: bool = True, - graphics: Optional[pg.GraphicObject] = None, ) -> list[QPainterPath]: ''' @@ -1153,23 +1133,32 @@ def render( hist = array[:slice_to_head] # last = array[slice_to_head] + # maybe allocate shm for data transform output + # if self.format_data is None: + # fshm = self.flow.shm + + # shm, opened = maybe_open_shm_array( + # f'{self.flow.name}_data_t', + # # TODO: create entry for each time frame + # dtype=array.dtype, + # readonly=False, + # ) + # assert opened + # shm.push(array) + # self.data_t_shm = shm + + # xy-path data transform: convert source data to a format + # able to be passed to a `QPainterPath` rendering routine. + # expected to be incrementally updates and later rendered to + # a more graphics native format. + # if self.data_t: + # array = self.data_t(array) ( x_out, y_out, - # x_iv, - # y_iv, connect, ) = self.format_xy(hist, array_key) - # print(f'{array_key} len x,y: {(len(x_out), len(y_out))}') -# # full input data -# x = array['index'] -# y = array[array_key] - -# # inview data -# x_iv = in_view['index'] -# y_iv = in_view[array_key] - profiler('sliced input arrays') ( @@ -1185,17 +1174,13 @@ def render( view_range = (ivl, ivr) # print(f'{self._name} vr: {view_range}') - # by default we only pull data up to the last (current) index - # x_out = x_iv[:slice_to_head] - # y_out = y_iv[:slice_to_head] - profiler(f'view range slice {view_range}') vl, vr = view_range zoom_or_append = False - last_vr = self._vr - last_ivr = self._avr + last_vr = self._last_vr + last_ivr = self._last_ivr # incremental in-view data update. if last_vr: @@ -1216,7 +1201,7 @@ def render( # underneath, so we need to redraw. # or left_change and right_change and last_vr == view_range - # not (left_change and right_change) and ivr + # not (left_change and right_change) and ivr # ( # or abs(x_iv[ivr] - livr) > 1 ): @@ -1232,40 +1217,28 @@ def render( should_redraw = True # print("REDRAWING BRUH") - self._vr = view_range + self._last_vr = view_range if len(x_out): - self._avr = x_out[0], x_out[slice_to_head] + self._last_ivr = x_out[0], x_out[slice_to_head] if prepend_length > 0: should_redraw = True - # # last datums - # x_last = x_out[-1] - # y_last = y_out[-1] - path = self.path fast_path = self.fast_path + # redraw the entire source data if we have either of: + # - no prior path graphic rendered or, + # - we always intend to re-render the data only in view if ( path is None or should_redraw or new_sample_rate or prepend_length > 0 ): - # if should_redraw: - # if path: - # path.clear() - # profiler('cleared paths due to `should_redraw=True`') - - # if fast_path: - # fast_path.clear() - - # profiler('cleared paths due to `should_redraw` set') if new_sample_rate and showing_src_data: - # if self._in_ds: log.info(f'DEDOWN -> {array_key}') - self._in_ds = False elif should_ds and uppx > 1: @@ -1277,8 +1250,6 @@ def render( ) profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True - # else: - # print(f"NOT DOWNSAMPLING {array_key}") path = self.draw_path( x=x_out, @@ -1287,15 +1258,6 @@ def render( path=path, redraw=True, ) - # path = pg.functions.arrayToQPath( - # x_out, - # y_out, - # connect='all', - # finiteCheck=False, - # path=path, - # ) - if graphics: - graphics.prepareGeometryChange() profiler( 'generated fresh path. ' @@ -1350,15 +1312,8 @@ def render( connect=connect, # path=fast_path, ) - - # append_path = pg.functions.arrayToQPath( - # connect='all', - # finiteCheck=False, - # path=fast_path, - # ) profiler('generated append qpath') - # if graphics.use_fpath: if use_fpath: print("USING FPATH") # an attempt at trying to make append-updates faster.. @@ -1381,51 +1336,6 @@ def render( profiler(f'connected history path w size: {size}') path.connectPath(append_path) - # if use_vr: - # array = in_view - # # get latest data from flow shm - # self.last_read = ( - # xfirst, xlast, array, ivl, ivr, in_view - # ) = new_read - - # if ( - # self.path is None - # or use_vr - # ): - # redraw the entire source data if we have either of: - # - no prior path graphic rendered or, - # - we always intend to re-render the data only in view - - # data transform: convert source data to a format - # expected to be incrementally updates and later rendered - # to a more graphics native format. - # if self.data_t: - # array = self.data_t(array) - - # maybe allocate shm for data transform output - # if self.data_t_shm is None: - # fshm = self.flow.shm - - # shm, opened = maybe_open_shm_array( - # f'{self.flow.name}_data_t', - # # TODO: create entry for each time frame - # dtype=array.dtype, - # readonly=False, - # ) - # assert opened - # shm.push(array) - # self.data_t_shm = shm - - # elif self.path: - # print(f'inremental update not supported yet {self.flow.name}') - # TODO: do incremental update - # prepend, append, last = self.diff(self.flow.read()) - - # do path generation for each segment - # and then push into graphics object. - - # call path render func on history - # self.path = self.draw_path(hist) self.path = path self.fast_path = fast_path From fa30df36ba840805ed11a7ee1573a4d1cfa3d449 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 19 May 2022 10:35:22 -0400 Subject: [PATCH 072/113] Simplify default xy formatter --- piker/ui/_flows.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index d7c5c2e63..282231e9f 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -925,15 +925,7 @@ def by_index_and_key( np.ndarray, np.ndarray, ]: - # full input data - x = array['index'] - y = array[array_key] - - return tuple({ - 'x': x, - 'y': y, - 'connect': 'all', - }.values()) + return array['index'], array[array_key], 'all' class Renderer(msgspec.Struct): @@ -941,7 +933,12 @@ class Renderer(msgspec.Struct): flow: Flow # last array view read last_read: Optional[tuple] = None - format_xy: Callable[np.ndarray, tuple[np.ndarray]] = by_index_and_key + + # default just returns index, and named array from data + format_xy: Callable[ + [np.ndarray, str], + tuple[np.ndarray] + ] = by_index_and_key # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` From 432d4545c22da356f8e31ee6fe89c61422f012ab Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 20 May 2022 16:52:44 -0400 Subject: [PATCH 073/113] Fix last values, must be pulled from source data in step mode --- piker/ui/_flows.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 282231e9f..c0da2739a 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -809,6 +809,8 @@ def update_graphics( # full input data x = array['index'] y = array[array_key] + x_last = x[-1] + y_last = y[-1] # inview data x_iv = in_view['index'] @@ -825,7 +827,7 @@ def update_graphics( ) profiler('generated step mode data') - ( + out = ( x, y, x_iv, @@ -842,9 +844,9 @@ def update_graphics( self._iflat_last, profiler, ) + input_data = out[:-1] - x_last = x[-1] - y_last = y[-1] + w = 0.5 graphics._last_line = QLineF( x_last - 0.5, 0, x_last + 0.5, 0, @@ -856,12 +858,6 @@ def update_graphics( should_redraw = bool(append_diff) draw_last = False - input_data = ( - x, - y, - x_iv, - y_iv, - ) # compute the length diffs between the first/last index entry in # the input data and the last indexes we have on record from the From f5de361f497190d0cdf553ab370b694ed50201f6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:44:20 -0400 Subject: [PATCH 074/113] Import directly from `tractor.trionics` --- piker/data/feed.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/data/feed.py b/piker/data/feed.py index d5e5d3b39..561d063ba 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -40,6 +40,7 @@ from trio_typing import TaskStatus import trimeter import tractor +from tractor.trionics import maybe_open_context from pydantic import BaseModel import pendulum import numpy as np From c256d3bdc0ea6eab37da525957d71482d563f0e9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:45:16 -0400 Subject: [PATCH 075/113] Type annot name in put to log routine --- piker/log.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/piker/log.py b/piker/log.py index 7c8bb798e..804e09dc6 100644 --- a/piker/log.py +++ b/piker/log.py @@ -25,10 +25,13 @@ # Makes it so we only see the full module name when using ``__name__`` # without the extra "piker." prefix. -_proj_name = 'piker' +_proj_name: str = 'piker' -def get_logger(name: str = None) -> logging.Logger: +def get_logger( + name: str = None, + +) -> logging.Logger: '''Return the package log or a sub-log for `name` if provided. ''' return tractor.log.get_logger(name=name, _root_name=_proj_name) From b985b48eb379e85bea0a03c1a52afc8419eaa9ad Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:45:44 -0400 Subject: [PATCH 076/113] Add `._last_bar_lines` guard to `.paint()` --- piker/ui/_ohlc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index fb57d6ff4..f10a49985 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -274,8 +274,9 @@ def paint( # lead to any perf gains other then when zoomed in to less bars # in view. p.setPen(self.last_bar_pen) - p.drawLines(*tuple(filter(bool, self._last_bar_lines))) - profiler('draw last bar') + if self._last_bar_lines: + p.drawLines(*tuple(filter(bool, self._last_bar_lines))) + profiler('draw last bar') p.setPen(self.bars_pen) p.drawPath(self.path) From 5d91516b41f4ec34bc8d4dfbdfebf2d8045a4bf8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 21 May 2022 11:46:56 -0400 Subject: [PATCH 077/113] Drop step mode "last datum" graphics creation from `.draw_last()` We're doing this in `Flow.update_graphics()` atm and probably are going to in general want custom graphics objects for all the diff curve / path types. The new flows work seems to fix the bounding rect width calcs to not require the ad-hoc extra `+ 1` in the step mode case; before it was always a bit hacky anyway. This also tries to add a more correct bounding rect adjustment for the `._last_line` segment. --- piker/ui/_curve.py | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index fa073d37d..4cffcd252 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -121,6 +121,7 @@ def __init__( self._last_line: Optional[QLineF] = None self._last_step_rect: Optional[QRectF] = None + self._last_w: float = 1 # flat-top style histogram-like discrete curve self._step_mode: bool = step_mode @@ -183,29 +184,11 @@ def draw_last( # draw the "current" step graphic segment so it lines up with # the "middle" of the current (OHLC) sample. - if self._step_mode: - self._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, - # x_last, 0, - # x_last, 0, - ) - self._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last - # x_last, 0, - # x_last, y_last - ) - # print( - # f"path br: {self.path.boundingRect()}", - # f"fast path br: {self.fast_path.boundingRect()}", - # f"last rect br: {self._last_step_rect}", - # ) - else: - self._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) + self._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + # self._last_w = x_last - x[-2] # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. @@ -290,13 +273,20 @@ def _path_br(self): # # hb_size.height() + 1 # ) - # if self._last_step_rect: # br = self._last_step_rect.bottomRight() - # else: - # hb_size += QSizeF(1, 1) - w = hb_size.width() + 1 - h = hb_size.height() + 1 + w = hb_size.width() + h = hb_size.height() + + if not self._last_step_rect: + # only on a plane line do we include + # and extra index step's worth of width + # since in the step case the end of the curve + # actually terminates earlier so we don't need + # this for the last step. + w += self._last_w + ll = self._last_line + h += ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From eca2401ab52ff77caf5367c04154622315a608d3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 08:55:45 -0400 Subject: [PATCH 078/113] Lul, well that heigh did not work.. --- piker/ui/_curve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 4cffcd252..abe489295 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -286,7 +286,7 @@ def _path_br(self): # this for the last step. w += self._last_w ll = self._last_line - h += ll.y2() - ll.y1() + h += 1 #ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From bbe1ff19ef1e94363447fb83cf95d539b7f06011 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 10:35:42 -0400 Subject: [PATCH 079/113] Don't kill all containers on teardown XD --- piker/data/_ahab.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py index 0f96ecaa2..fea19a4d4 100644 --- a/piker/data/_ahab.py +++ b/piker/data/_ahab.py @@ -98,8 +98,6 @@ def unpack_msg(err: Exception) -> str: finally: if client: client.close() - for c in client.containers.list(): - c.kill() class Container: From 1b38628b09c853ca906cb541f48dab5f04d4f1c6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 10:36:17 -0400 Subject: [PATCH 080/113] Handle teardown race, add comment about shm subdirs --- piker/data/_sharedmem.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 8848ec1c3..47d58d3e6 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -98,7 +98,12 @@ def destroy(self) -> None: if _USE_POSIX: # We manually unlink to bypass all the "resource tracker" # nonsense meant for non-SC systems. - shm_unlink(self._shm.name) + name = self._shm.name + try: + shm_unlink(name) + except FileNotFoundError: + # might be a teardown race here? + log.warning(f'Shm for {name} already unlinked?') class _Token(BaseModel): @@ -536,6 +541,10 @@ def attach_shm_array( if key in _known_tokens: assert _Token.from_msg(_known_tokens[key]) == token, "WTF" + # XXX: ugh, looks like due to the ``shm_open()`` C api we can't + # actually place files in a subdir, see discussion here: + # https://stackoverflow.com/a/11103289 + # attach to array buffer and view as per dtype shm = SharedMemory(name=key) shmarr = np.ndarray( From 8ce7e99210db10c5bb41797183d5571ff0f37eed Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 11:41:08 -0400 Subject: [PATCH 081/113] Drop prints --- piker/ui/_flows.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c0da2739a..1219627aa 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -1281,8 +1281,7 @@ def render( and do_append and not should_redraw ): - # print(f'{self.name} append len: {append_length}') - print(f'{array_key} append len: {append_length}') + # print(f'{array_key} append len: {append_length}') new_x = x_out[-append_length - 2:] # slice_to_head] new_y = y_out[-append_length - 2:] # slice_to_head] profiler('sliced append path') @@ -1308,7 +1307,6 @@ def render( profiler('generated append qpath') if use_fpath: - print("USING FPATH") # an attempt at trying to make append-updates faster.. if fast_path is None: fast_path = append_path From 42572d38087919daa079e3517a4cc74596c1b426 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 24 May 2022 14:22:30 -0400 Subject: [PATCH 082/113] Add back linked plots/views y-range autoscaling --- piker/ui/_interaction.py | 70 +++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 90242c999..8e95855e9 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -570,6 +570,13 @@ def wheelEvent( self._resetTarget() self.scaleBy(s, focal) + # XXX: the order of the next 2 lines i'm pretty sure + # matters, we want the resize to trigger before the graphics + # update, but i gotta feelin that because this one is signal + # based (and thus not necessarily sync invoked right away) + # that calling the resize method manually might work better. + self.sigRangeChangedManually.emit(mask) + # XXX: without this is seems as though sometimes # when zooming in from far out (and maybe vice versa?) # the signal isn't being fired enough since if you pan @@ -580,12 +587,6 @@ def wheelEvent( # fires don't happen? self.maybe_downsample_graphics() - self.sigRangeChangedManually.emit(mask) - - # self._ic.set() - # self._ic = None - # self.chart.resume_all_feeds() - ev.accept() def mouseDragEvent( @@ -746,7 +747,7 @@ def _set_yrange( # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = False, + autoscale_linked_plots: bool = True, name: Optional[str] = None, # autoscale_overlays: bool = False, @@ -759,6 +760,7 @@ def _set_yrange( data set. ''' + # print(f'YRANGE ON {self.name}') profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -788,42 +790,11 @@ def _set_yrange( elif yrange is not None: ylow, yhigh = yrange - # calculate max, min y values in viewable x-range from data. - # Make sure min bars/datums on screen is adhered. - # else: - # TODO: eventually we should point to the - # ``FlowsTable`` (or wtv) which should perform - # the group operations? - - # flow = chart._flows[name or chart.name] - # br = bars_range or chart.bars_range() - # br = bars_range or chart.bars_range() - # profiler(f'got bars range: {br}') - - # TODO: maybe should be a method on the - # chart widget/item? - # if False: - # if autoscale_linked_plots: - # # avoid recursion by sibling plots - # linked = self.linkedsplits - # plots = list(linked.subplots.copy().values()) - # main = linked.chart - # if main: - # plots.append(main) - - # for chart in plots: - # if chart and not chart._static_yrange: - # chart.cv._set_yrange( - # # bars_range=br, - # autoscale_linked_plots=False, - # ) - # profiler('autoscaled linked plots') - if set_range: + # XXX: only compute the mxmn range + # if none is provided as input! if not yrange: - # XXX: only compute the mxmn range - # if none is provided as input! yrange = self._maxmin() if yrange is None: @@ -850,6 +821,25 @@ def _set_yrange( self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') + # TODO: maybe should be a method on the + # chart widget/item? + if autoscale_linked_plots: + # avoid recursion by sibling plots + linked = self.linkedsplits + plots = list(linked.subplots.copy().values()) + main = linked.chart + if main: + plots.append(main) + + # print(f'autoscaling linked: {plots}') + for chart in plots: + if chart and not chart._static_yrange: + chart.cv._set_yrange( + # bars_range=br, + autoscale_linked_plots=False, + ) + profiler('autoscaled linked plots') + profiler.finish() def enable_auto_yrange( From 04897fd402024cdbd63ca594e9009bc5f0831a46 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 11:15:46 -0400 Subject: [PATCH 083/113] Implement pre-graphics format incremental update Adds a new pre-graphics data-format callback incremental update api to our `Renderer`. `Renderer` instance can now overload these custom routines: - `.update_xy()` a routine which accepts the latest [pre/a]pended data sliced out from shm and returns it in a format suitable to store in the optional `.[x/y]_data` arrays. - `.allocate_xy()` which initially does the work of pre-allocating the `.[x/y]_data` arrays based on the source shm sizing such that new data can be filled in (to memory). - `._xy_[first/last]: int` attrs to track index diffs between src shm and the xy format data updates. Implement the step curve data format with 3 super simple routines: - `.allocate_xy()` -> `._pathops.to_step_format()` - `.update_xy()` -> `._flows.update_step_xy()` - `.format_xy()` -> `._flows.step_to_xy()` Further, adjust `._pathops.gen_ohlc_qpath()` to adhere to the new call signature. --- piker/ui/_flows.py | 410 +++++++++++++++++++++---------------------- piker/ui/_pathops.py | 16 +- 2 files changed, 216 insertions(+), 210 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 1219627aa..043f9243f 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -144,8 +144,9 @@ def render_baritems( # if no source data renderer exists create one. self = flow - r = self._src_r show_bars: bool = False + + r = self._src_r if not r: show_bars = True # OHLC bars path renderer @@ -188,7 +189,7 @@ def render_baritems( # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to # render the bars graphics curve and update.. - # - if insteam we are in a downsamplig state then we to + # - if instead we are in a downsamplig state then we to x_gt = 6 uppx = curve.x_uppx() in_line = should_line = curve.isVisible() @@ -212,6 +213,7 @@ def render_baritems( if should_line: fields = ['open', 'high', 'low', 'close'] + if self.gy is None: # create a flattened view onto the OHLC array # which can be read as a line-style format @@ -373,119 +375,75 @@ def render_baritems( ) -def update_step_data( - flow: Flow, - shm: ShmArray, - ivl: int, - ivr: int, +def update_step_xy( + src_shm: ShmArray, array_key: str, - iflat_first: int, - iflat: int, - profiler: pg.debug.Profiler, + y_update: np.ndarray, + slc: slice, + ln: int, + first: int, + last: int, + is_append: bool, + +) -> np.ndarray: + + # for a step curve we slice from one datum prior + # to the current "update slice" to get the previous + # "level". + if is_append: + start = max(last - 1, 0) + end = src_shm._last.value + new_y = src_shm._array[start:end][array_key] + slc = slice(start, end) -) -> tuple: + else: + new_y = y_update - self = flow - ( - # iflat_first, - # iflat, - ishm_last, - ishm_first, - ) = ( - # self._iflat_first, - # self._iflat_last, - shm._last.value, - shm._first.value + return ( + np.broadcast_to( + new_y[:, None], (new_y.size, 2), + ), + slc, ) - il = max(iflat - 1, 0) - profiler('read step mode incr update indices') - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - print(f'prepend {array_key}') - - # i_prepend = self.shm._array['index'][ - # ishm_first:iflat_first] - y_prepend = self.shm._array[array_key][ - ishm_first:iflat_first - ] - - y2_prepend = np.broadcast_to( - y_prepend[:, None], (y_prepend.size, 2), - ) - - # write newly prepended data to flattened copy - self.gy[ishm_first:iflat_first] = y2_prepend - self._iflat_first = ishm_first - profiler('prepended step mode history') - - append_diff = ishm_last - iflat - if append_diff: - - # slice up to the last datum since last index/append update - # new_x = self.shm._array[il:ishm_last]['index'] - new_y = self.shm._array[il:ishm_last][array_key] - - new_y2 = np.broadcast_to( - new_y[:, None], (new_y.size, 2), - ) - self.gy[il:ishm_last] = new_y2 - profiler('updated step curve data') - - # print( - # f'append size: {append_diff}\n' - # f'new_x: {new_x}\n' - # f'new_y: {new_y}\n' - # f'new_y2: {new_y2}\n' - # f'new gy: {gy}\n' - # ) +def step_to_xy( + r: Renderer, + array: np.ndarray, + array_key: str, + vr: tuple[int, int], - # update local last-index tracking - self._iflat_last = ishm_last +) -> tuple[ + np.ndarray, + np.nd.array, + str, +]: - # slice out up-to-last step contents - x_step = self.gx[ishm_first:ishm_last+2] - # shape to 1d - x = x_step.reshape(-1) - profiler('sliced step x') + # 2 more datum-indexes to capture zero at end + x_step = r.x_data[r._xy_first:r._xy_last+2] + y_step = r.y_data[r._xy_first:r._xy_last+2] - y_step = self.gy[ishm_first:ishm_last+2] - lasts = self.shm.array[['index', array_key]] + lasts = array[['index', array_key]] last = lasts[array_key][-1] y_step[-1] = last - # shape to 1d - y = y_step.reshape(-1) - # s = 6 - # print(f'lasts: {x[-2*s:]}, {y[-2*s:]}') - - profiler('sliced step y') - - # do all the same for only in-view data + # slice out in-view data + ivl, ivr = vr ys_iv = y_step[ivl:ivr+1] xs_iv = x_step[ivl:ivr+1] + + # flatten to 1d y_iv = ys_iv.reshape(ys_iv.size) x_iv = xs_iv.reshape(xs_iv.size) + # print( # f'ys_iv : {ys_iv[-s:]}\n' # f'y_iv: {y_iv[-s:]}\n' # f'xs_iv: {xs_iv[-s:]}\n' # f'x_iv: {x_iv[-s:]}\n' # ) - profiler('sliced in view step data') - # legacy full-recompute-everytime method - # x, y = ohlc_flatten(array) - # x_iv, y_iv = ohlc_flatten(in_view) - # profiler('flattened OHLC data') - return ( - x, - y, - x_iv, - y_iv, - append_diff, - ) + return x_iv, y_iv, 'all' class Flow(msgspec.Struct): # , frozen=True): @@ -508,7 +466,7 @@ class Flow(msgspec.Struct): # , frozen=True): render: bool = True # toggle for display loop # pre-graphics formatted data - gy: Optional[ShmArray] = None + gy: Optional[np.ndarray] = None gx: Optional[np.ndarray] = None # pre-graphics update indices @@ -723,9 +681,9 @@ def update_graphics( out: Optional[tuple] = None if isinstance(graphics, BarItems): draw_last = False + # XXX: special case where we change out graphics # to a line after a certain uppx threshold. - # render_baritems( out = render_baritems( self, graphics, @@ -739,19 +697,8 @@ def update_graphics( # return graphics - r = self._src_r - if not r: - # just using for ``.diff()`` atm.. - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - # draw_path=gen_ohlc_qpath, - last_read=read, - ) - # ``FastAppendCurve`` case: array_key = array_key or self.name - shm = self.shm if out is not None: # hack to handle ds curve from bars above @@ -763,7 +710,49 @@ def update_graphics( y_iv, ) = out input_data = out[1:] - # breakpoint() + + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + # draw_path=gen_ohlc_qpath, + last_read=read, + ) + + if graphics._step_mode: + + r.allocate_xy = to_step_format + r.update_xy = update_step_xy + r.format_xy = step_to_xy + + slice_to_head = -2 + + # TODO: append logic inside ``.render()`` isn't + # corrent yet for step curves.. remove this to see it. + should_redraw = True + + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = array['index'] + y = array[array_key] + x_last = x[-1] + y_last = y[-1] + + w = 0.5 + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + # should_redraw = bool(append_diff) + draw_last = False # ds update config new_sample_rate: bool = False @@ -780,7 +769,7 @@ def update_graphics( uppx > 1 and abs(uppx_diff) >= 1 ): - log.info( + log.debug( f'{array_key} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx @@ -801,69 +790,6 @@ def update_graphics( should_ds = False showing_src_data = True - if graphics._step_mode: - slice_to_head = -2 - - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = array['index'] - y = array[array_key] - x_last = x[-1] - y_last = y[-1] - - # inview data - x_iv = in_view['index'] - y_iv = in_view[array_key] - - if self.gy is None: - ( - self._iflat_first, - self.gx, - self.gy, - ) = to_step_format( - shm, - array_key, - ) - profiler('generated step mode data') - - out = ( - x, - y, - x_iv, - y_iv, - append_diff, - - ) = update_step_data( - self, - shm, - ivl, - ivr, - array_key, - self._iflat_first, - self._iflat_last, - profiler, - ) - input_data = out[:-1] - - w = 0.5 - graphics._last_line = QLineF( - x_last - 0.5, 0, - x_last + 0.5, 0, - ) - graphics._last_step_rect = QRectF( - x_last - 0.5, 0, - x_last + 0.5, y_last, - ) - - should_redraw = bool(append_diff) - draw_last = False - - # compute the length diffs between the first/last index entry in - # the input data and the last indexes we have on record from the - # last time we updated the curve index. - # prepend_length, append_length = r.diff(read) - # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed @@ -913,8 +839,10 @@ def update_graphics( def by_index_and_key( + renderer: Renderer, array: np.ndarray, array_key: str, + vr: tuple[int, int], ) -> tuple[ np.ndarray, @@ -936,15 +864,31 @@ class Renderer(msgspec.Struct): tuple[np.ndarray] ] = by_index_and_key + # optional pre-graphics xy formatted data which + # is incrementally updated in sync with the source data. + allocate_xy: Optional[Callable[ + [int, slice], + tuple[np.ndarray, np.nd.array] + ]] = None + + update_xy: Optional[Callable[ + [int, slice], None] + ] = None + + x_data: Optional[np.ndarray] = None + y_data: Optional[np.ndarray] = None + + # indexes which slice into the above arrays (which are allocated + # based on source data shm input size) and allow retrieving + # incrementally updated data. + _xy_first: int = 0 + _xy_last: int = 0 + # output graphics rendering, the main object # processed in ``QGraphicsObject.paint()`` path: Optional[QPainterPath] = None fast_path: Optional[QPainterPath] = None - # called on input data but before any graphics format - # conversions or processing. - format_data: Optional[Callable[ShmArray, np.ndarray]] = None - # XXX: just ideas.. # called on the final data (transform) output to convert # to "graphical data form" a format that can be passed to @@ -998,17 +942,13 @@ def diff( prepend_length = int(last_xfirst - xfirst) append_length = int(xlast - last_xlast) - # TODO: eventually maybe we can implement some kind of - # transform on the ``QPainterPath`` that will more or less - # detect the diff in "elements" terms? - # update state - self.last_read = new_read - # blah blah blah # do diffing for prepend, append and last entry return ( + slice(xfirst, last_xfirst), prepend_length, append_length, + slice(last_xlast, xlast), ) def draw_path( @@ -1103,6 +1043,75 @@ def render( in_view, ) = new_read + ( + pre_slice, + prepend_length, + append_length, + post_slice, + ) = self.diff(new_read) + + if self.update_xy: + + shm = self.flow.shm + + if self.y_data is None: + # we first need to allocate xy data arrays + # from the source data. + assert self.allocate_xy + self.x_data, self.y_data = self.allocate_xy( + shm, + array_key, + ) + self._xy_first = shm._first.value + self._xy_last = shm._last.value + profiler('allocated xy history') + + if prepend_length: + y_prepend = shm._array[array_key][pre_slice] + + xy_data, xy_slice = self.update_xy( + shm, + array_key, + + # this is the pre-sliced, "normally expected" + # new data that an updater would normally be + # expected to process, however in some cases (like + # step curves) the updater routine may want to do + # the source history-data reading itself, so we pass + # both here. + y_prepend, + + pre_slice, + prepend_length, + self._xy_first, + self._xy_last, + is_append=False, + ) + self.y_data[xy_slice] = xy_data + self._xy_first = shm._first.value + profiler('prepended xy history: {prepend_length}') + + if append_length: + y_append = shm._array[array_key][post_slice] + + xy_data, xy_slice = self.update_xy( + shm, + array_key, + + y_append, + post_slice, + append_length, + + self._xy_first, + self._xy_last, + is_append=True, + ) + # self.y_data[post_slice] = xy_data + # self.y_data[xy_slice or post_slice] = xy_data + self.y_data[xy_slice] = xy_data + self._xy_last = shm._last.value + profiler('appened xy history: {append_length}') + if use_vr: array = in_view @@ -1120,45 +1129,31 @@ def render( x_out = x_iv y_out = y_iv - # last = y_out[slice_to_head] - else: - hist = array[:slice_to_head] - # last = array[slice_to_head] - - # maybe allocate shm for data transform output - # if self.format_data is None: - # fshm = self.flow.shm - - # shm, opened = maybe_open_shm_array( - # f'{self.flow.name}_data_t', - # # TODO: create entry for each time frame - # dtype=array.dtype, - # readonly=False, - # ) - # assert opened - # shm.push(array) - # self.data_t_shm = shm - # xy-path data transform: convert source data to a format # able to be passed to a `QPainterPath` rendering routine. # expected to be incrementally updates and later rendered to # a more graphics native format. # if self.data_t: # array = self.data_t(array) + + hist = array[:slice_to_head] ( x_out, y_out, connect, - ) = self.format_xy(hist, array_key) + ) = self.format_xy( + self, + # TODO: hist here should be the pre-sliced + # x/y_data in the case where allocate_xy is + # defined? + hist, + array_key, + (ivl, ivr), + ) profiler('sliced input arrays') - ( - prepend_length, - append_length, - ) = self.diff(new_read) - if ( use_vr ): @@ -1330,5 +1325,10 @@ def render( self.path = path self.fast_path = fast_path + # TODO: eventually maybe we can implement some kind of + # transform on the ``QPainterPath`` that will more or less + # detect the diff in "elements" terms? + # update diff state since we've now rendered paths. self.last_read = new_read + return self.path, array diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 4cb5b86e1..89f7c5dc9 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -17,25 +17,30 @@ Super fast ``QPainterPath`` generation related operator routines. """ +from __future__ import annotations from typing import ( - Optional, + # Optional, + TYPE_CHECKING, ) import numpy as np from numpy.lib import recfunctions as rfn from numba import njit, float64, int64 # , optional -import pyqtgraph as pg +# import pyqtgraph as pg from PyQt5 import QtGui # from PyQt5.QtCore import QLineF, QPointF from ..data._sharedmem import ( ShmArray, ) -from .._profile import pg_profile_enabled, ms_slower_then +# from .._profile import pg_profile_enabled, ms_slower_then from ._compression import ( ds_m4, ) +if TYPE_CHECKING: + from ._flows import Renderer + def xy_downsample( x, @@ -138,8 +143,10 @@ def path_arrays_from_ohlc( def gen_ohlc_qpath( + r: Renderer, data: np.ndarray, array_key: str, # we ignore this + vr: tuple[int, int], start: int = 0, # XXX: do we need this? # 0.5 is no overlap between arms, 1.0 is full overlap @@ -216,7 +223,6 @@ def to_step_format( for use by path graphics generation. ''' - first = shm._first.value i = shm._array['index'].copy() out = shm._array[data_field].copy() @@ -230,4 +236,4 @@ def to_step_format( # start y at origin level y_out[0, 0] = 0 - return first, x_out, y_out + return x_out, y_out From d4f31f2b3c569337c67d6cc72f683bfd3794dedc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 11:41:52 -0400 Subject: [PATCH 084/113] Move update-state-vars defaults above step mode block --- piker/ui/_flows.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 043f9243f..7714dd67e 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -721,6 +721,12 @@ def update_graphics( last_read=read, ) + # ds update config + new_sample_rate: bool = False + should_redraw: bool = False + should_ds: bool = r._in_ds + showing_src_data: bool = not r._in_ds + if graphics._step_mode: r.allocate_xy = to_step_format @@ -754,12 +760,6 @@ def update_graphics( # should_redraw = bool(append_diff) draw_last = False - # ds update config - new_sample_rate: bool = False - should_redraw: bool = False - should_ds: bool = r._in_ds - showing_src_data: bool = not r._in_ds - # downsampling incremental state checking # check for and set std m4 downsample conditions uppx = graphics.x_uppx() From 066b8df619dc820ab8079e135bf429246abc2a5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 25 May 2022 19:45:09 -0400 Subject: [PATCH 085/113] Implement OHLC downsampled curve via renderer, drop old bypass code --- piker/ui/_flows.py | 434 +++++++++++++++---------------------------- piker/ui/_pathops.py | 5 +- 2 files changed, 154 insertions(+), 285 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 7714dd67e..e74ee123f 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,7 @@ ''' from __future__ import annotations -# from functools import partial +from functools import partial from typing import ( Optional, Callable, @@ -45,7 +45,6 @@ from ..data._sharedmem import ( ShmArray, - # open_shm_array, ) from .._profile import ( pg_profile_enabled, @@ -68,6 +67,7 @@ log = get_logger(__name__) + # class FlowsTable(msgspec.Struct): # ''' # Data-AGGRegate: high level API onto multiple (categorized) @@ -77,42 +77,54 @@ # ''' # flows: dict[str, np.ndarray] = {} -# @classmethod -# def from_token( -# cls, -# shm_token: tuple[ -# str, -# str, -# tuple[str, str], -# ], -# ) -> Renderer: +def update_ohlc_to_line( + src_shm: ShmArray, + array_key: str, + src_update: np.ndarray, + slc: slice, + ln: int, + first: int, + last: int, + is_append: bool, -# shm = attach_shm_array(token) -# return cls(shm) +) -> np.ndarray: + fields = ['open', 'high', 'low', 'close'] + return ( + rfn.structured_to_unstructured(src_update[fields]), + slc, + ) -def rowarr_to_path( - rows_array: np.ndarray, - x_basis: np.ndarray, - flow: Flow, -) -> QPainterPath: +def ohlc_flat_to_xy( + r: Renderer, + array: np.ndarray, + array_key: str, + vr: tuple[int, int], - # TODO: we could in theory use ``numba`` to flatten - # if needed? +) -> tuple[ + np.ndarray, + np.nd.array, + str, +]: + # TODO: in the case of an existing ``.update_xy()`` + # should we be passing in array as an xy arrays tuple? - # to 1d - y = rows_array.flatten() + # 2 more datum-indexes to capture zero at end + x_flat = r.x_data[r._xy_first:r._xy_last] + y_flat = r.y_data[r._xy_first:r._xy_last] - return pg.functions.arrayToQPath( - # these get passed at render call time - x=x_basis[:y.size], - y=y, - connect='all', - finiteCheck=False, - path=flow.path, - ) + # slice to view + ivl, ivr = vr + x_iv_flat = x_flat[ivl:ivr] + y_iv_flat = y_flat[ivl:ivr] + + # reshape to 1d for graphics rendering + y_iv = y_iv_flat.reshape(-1) + x_iv = x_iv_flat.reshape(-1) + + return x_iv, y_iv, 'all' def render_baritems( @@ -137,10 +149,7 @@ def render_baritems( layer, if not a `Renderer` then something just above it? ''' - ( - xfirst, xlast, array, - ivl, ivr, in_view, - ) = read + bars = graphics # if no source data renderer exists create one. self = flow @@ -156,35 +165,28 @@ def render_baritems( last_read=read, ) - # ds_curve_r = Renderer( - # flow=self, + ds_curve_r = Renderer( + flow=self, + last_read=read, - # # just swap in the flat view - # # data_t=lambda array: self.gy.array, - # last_read=read, - # draw_path=partial( - # rowarr_to_path, - # x_basis=None, - # ), + # incr update routines + allocate_xy=ohlc_to_line, + update_xy=update_ohlc_to_line, + format_xy=ohlc_flat_to_xy, + ) - # ) curve = FastAppendCurve( - name='OHLC', - color=graphics._color, + name=f'{flow.name}_ds_ohlc', + color=bars._color, ) curve.hide() self.plot.addItem(curve) # baseline "line" downsampled OHLC curve that should # kick on only when we reach a certain uppx threshold. - self._render_table[0] = curve - # ( - # # ds_curve_r, - # curve, - # ) + self._render_table = (ds_curve_r, curve) - curve = self._render_table[0] - # dsc_r, curve = self._render_table[0] + ds_r, curve = self._render_table # do checks for whether or not we require downsampling: # - if we're **not** downsampling then we simply want to @@ -197,183 +199,74 @@ def render_baritems( should_line and uppx < x_gt ): - print('FLIPPING TO BARS') + # print('FLIPPING TO BARS') should_line = False elif ( not should_line and uppx >= x_gt ): - print('FLIPPING TO LINE') + # print('FLIPPING TO LINE') should_line = True profiler(f'ds logic complete line={should_line}') # do graphics updates if should_line: - - fields = ['open', 'high', 'low', 'close'] - - if self.gy is None: - # create a flattened view onto the OHLC array - # which can be read as a line-style format - shm = self.shm - ( - self._iflat_first, - self._iflat_last, - self.gx, - self.gy, - ) = ohlc_to_line( - shm, - fields=fields, - ) - - # print(f'unstruct diff: {time.time() - start}') - - gy = self.gy - - # update flatted ohlc copy - ( - iflat_first, - iflat, - ishm_last, - ishm_first, - ) = ( - self._iflat_first, - self._iflat_last, - self.shm._last.value, - self.shm._first.value - ) - - # check for shm prepend updates since last read. - if iflat_first != ishm_first: - - # write newly prepended data to flattened copy - gy[ - ishm_first:iflat_first - ] = rfn.structured_to_unstructured( - self.shm._array[fields][ishm_first:iflat_first] - ) - self._iflat_first = ishm_first - - to_update = rfn.structured_to_unstructured( - self.shm._array[iflat:ishm_last][fields] - ) - - gy[iflat:ishm_last][:] = to_update - profiler('updated ustruct OHLC data') - - # slice out up-to-last step contents - y_flat = gy[ishm_first:ishm_last] - x_flat = self.gx[ishm_first:ishm_last] - - # update local last-index tracking - self._iflat_last = ishm_last - - # reshape to 1d for graphics rendering - y = y_flat.reshape(-1) - x = x_flat.reshape(-1) - profiler('flattened ustruct OHLC data') - - # do all the same for only in-view data - y_iv_flat = y_flat[ivl:ivr] - x_iv_flat = x_flat[ivl:ivr] - y_iv = y_iv_flat.reshape(-1) - x_iv = x_iv_flat.reshape(-1) - profiler('flattened ustruct in-view OHLC data') - - # pass into curve graphics processing - # curve.update_from_array( - # x, - # y, - # x_iv=x_iv, - # y_iv=y_iv, - # view_range=(ivl, ivr), # hack - # profiler=profiler, - # # should_redraw=False, - - # # NOTE: already passed through by display loop? - # # do_append=uppx < 16, - # **kwargs, - # ) - curve.draw_last(x, y) - curve.show() + r = ds_r + graphics = curve profiler('updated ds curve') else: - # render incremental or in-view update - # and apply ouput (path) to graphics. - path, data = r.render( - read, - 'ohlc', - profiler=profiler, - # uppx=1, - use_vr=True, - # graphics=graphics, - # should_redraw=True, # always - ) - assert path - - graphics.path = path - graphics.draw_last(data[-1]) - if show_bars: - graphics.show() + graphics = bars - # NOTE: on appends we used to have to flip the coords - # cache thought it doesn't seem to be required any more? - # graphics.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # graphics.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # graphics.prepareGeometryChange() - graphics.update() + if show_bars: + bars.show() + changed_to_line = False if ( not in_line and should_line ): # change to line graphic - log.info( f'downsampling to line graphic {self.name}' ) - graphics.hide() - # graphics.update() + bars.hide() curve.show() curve.update() + changed_to_line = True elif in_line and not should_line: + # change to bars graphic log.info(f'showing bars graphic {self.name}') curve.hide() - graphics.show() - graphics.update() + bars.show() + bars.update() - # update our pre-downsample-ready data and then pass that - # new data the downsampler algo for incremental update. - - # graphics.update_from_array( - # array, - # in_view, - # view_range=(ivl, ivr) if use_vr else None, - - # **kwargs, - # ) - - # generate and apply path to graphics obj - # graphics.path, last = r.render( - # read, - # only_in_view=True, - # ) - # graphics.draw_last(last) + draw_last = False + lasts = self.shm.array[-2:] + last = lasts[-1] if should_line: - return ( - curve, - x, - y, - x_iv, - y_iv, + def draw_last(): + x, y = lasts['index'], lasts['close'] + curve.draw_last(x, y) + else: + draw_last = partial( + bars.draw_last, + last, ) + return ( + graphics, + r, + {'read_from_key': False}, + draw_last, + should_line, + changed_to_line, + ) + def update_step_xy( src_shm: ShmArray, @@ -484,7 +377,7 @@ class Flow(msgspec.Struct): # , frozen=True): _render_table: dict[ Optional[int], tuple[Renderer, pg.GraphicsItem], - ] = {} + ] = (None, None) # TODO: hackery to be able to set a shm later # but whilst also allowing this type to hashable, @@ -672,62 +565,58 @@ def update_graphics( not in_view.size or not render ): + # print('exiting early') return graphics draw_last: bool = True slice_to_head: int = -1 - input_data = None + # input_data = None - out: Optional[tuple] = None - if isinstance(graphics, BarItems): - draw_last = False + should_redraw: bool = False + + rkwargs = {} + bars = False + if isinstance(graphics, BarItems): # XXX: special case where we change out graphics # to a line after a certain uppx threshold. - out = render_baritems( + ( + graphics, + r, + rkwargs, + draw_last, + should_line, + changed_to_line, + ) = render_baritems( self, graphics, read, profiler, **kwargs, ) + bars = True + should_redraw = changed_to_line or not should_line - if out is None: - return graphics - - # return graphics + else: + r = self._src_r + if not r: + # just using for ``.diff()`` atm.. + r = self._src_r = Renderer( + flow=self, + # TODO: rename this to something with ohlc + last_read=read, + ) # ``FastAppendCurve`` case: array_key = array_key or self.name - - if out is not None: - # hack to handle ds curve from bars above - ( - graphics, # curve - x, - y, - x_iv, - y_iv, - ) = out - input_data = out[1:] - - r = self._src_r - if not r: - # just using for ``.diff()`` atm.. - r = self._src_r = Renderer( - flow=self, - # TODO: rename this to something with ohlc - # draw_path=gen_ohlc_qpath, - last_read=read, - ) + # print(array_key) # ds update config new_sample_rate: bool = False - should_redraw: bool = False should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - if graphics._step_mode: + if getattr(graphics, '_step_mode', False): r.allocate_xy = to_step_format r.update_xy = update_step_xy @@ -756,8 +645,6 @@ def update_graphics( x_last - w, 0, x_last + w, y_last, ) - - # should_redraw = bool(append_diff) draw_last = False # downsampling incremental state checking @@ -795,15 +682,11 @@ def update_graphics( # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - # path = graphics.path - # fast_path = graphics.fast_path - path, data = r.render( read, array_key, profiler, uppx=uppx, - input_data=input_data, # use_vr=True, # TODO: better way to detect and pass this? @@ -816,6 +699,8 @@ def update_graphics( slice_to_head=slice_to_head, do_append=do_append, + + **rkwargs, ) # TODO: does this actuallly help us in any way (prolly should # look at the source / ask ogi). @@ -825,11 +710,16 @@ def update_graphics( graphics.path = r.path graphics.fast_path = r.fast_path - if draw_last: + if draw_last and not bars: + # TODO: how to handle this draw last stuff.. x = data['index'] y = data[array_key] graphics.draw_last(x, y) - profiler('draw last segment') + + elif bars and draw_last: + draw_last() + + profiler('draw last segment') graphics.update() profiler('.update()') @@ -1004,10 +894,8 @@ def render( profiler: pg.debug.Profiler, uppx: float = 1, - input_data: Optional[tuple[np.ndarray]] = None, - # redraw and ds flags - should_redraw: bool = True, + should_redraw: bool = False, new_sample_rate: bool = False, should_ds: bool = False, showing_src_data: bool = True, @@ -1018,6 +906,7 @@ def render( # only render datums "in view" of the ``ChartView`` use_vr: bool = True, + read_from_key: bool = True, ) -> list[QPainterPath]: ''' @@ -1067,7 +956,10 @@ def render( profiler('allocated xy history') if prepend_length: - y_prepend = shm._array[array_key][pre_slice] + y_prepend = shm._array[pre_slice] + + if read_from_key: + y_prepend = y_prepend[array_key] xy_data, xy_slice = self.update_xy( shm, @@ -1092,7 +984,10 @@ def render( profiler('prepended xy history: {prepend_length}') if append_length: - y_append = shm._array[array_key][post_slice] + y_append = shm._array[post_slice] + + if read_from_key: + y_append = y_append[array_key] xy_data, xy_slice = self.update_xy( shm, @@ -1114,43 +1009,22 @@ def render( if use_vr: array = in_view - - if input_data: - # allow input data passing for now from alt curve updaters. - ( - x_out, - y_out, - x_iv, - y_iv, - ) = input_data - connect = 'all' - - if use_vr: - x_out = x_iv - y_out = y_iv - else: - # xy-path data transform: convert source data to a format - # able to be passed to a `QPainterPath` rendering routine. - # expected to be incrementally updates and later rendered to - # a more graphics native format. - # if self.data_t: - # array = self.data_t(array) - - hist = array[:slice_to_head] - ( - x_out, - y_out, - connect, - ) = self.format_xy( - self, - # TODO: hist here should be the pre-sliced - # x/y_data in the case where allocate_xy is - # defined? - hist, - array_key, - (ivl, ivr), - ) + ivl, ivr = xfirst, xlast + + hist = array[:slice_to_head] + + # xy-path data transform: convert source data to a format + # able to be passed to a `QPainterPath` rendering routine. + x_out, y_out, connect = self.format_xy( + self, + # TODO: hist here should be the pre-sliced + # x/y_data in the case where allocate_xy is + # defined? + hist, + array_key, + (ivl, ivr), + ) profiler('sliced input arrays') @@ -1168,7 +1042,7 @@ def render( zoom_or_append = False last_vr = self._last_vr - last_ivr = self._last_ivr + last_ivr = self._last_ivr or vl, vr # incremental in-view data update. if last_vr: @@ -1203,7 +1077,6 @@ def render( ) ): should_redraw = True - # print("REDRAWING BRUH") self._last_vr = view_range if len(x_out): @@ -1224,7 +1097,7 @@ def render( or new_sample_rate or prepend_length > 0 ): - + # print("REDRAWING BRUH") if new_sample_rate and showing_src_data: log.info(f'DEDOWN -> {array_key}') self._in_ds = False @@ -1252,7 +1125,6 @@ def render( f'(should_redraw: {should_redraw} ' f'should_ds: {should_ds} new_sample_rate: {new_sample_rate})' ) - # profiler(f'DRAW PATH IN VIEW -> {self.name}') # TODO: get this piecewise prepend working - right now it's # giving heck on vwap... @@ -1297,7 +1169,7 @@ def render( x=new_x, y=new_y, connect=connect, - # path=fast_path, + path=fast_path, ) profiler('generated append qpath') diff --git a/piker/ui/_pathops.py b/piker/ui/_pathops.py index 89f7c5dc9..83b46f43e 100644 --- a/piker/ui/_pathops.py +++ b/piker/ui/_pathops.py @@ -168,11 +168,10 @@ def gen_ohlc_qpath( def ohlc_to_line( ohlc_shm: ShmArray, + data_field: str, fields: list[str] = ['open', 'high', 'low', 'close'] ) -> tuple[ - int, # flattened first index - int, # flattened last index np.ndarray, np.ndarray, ]: @@ -205,8 +204,6 @@ def ohlc_to_line( assert y_out.any() return ( - first, - last, x_out, y_out, ) From 08c83afa9006c4e7516f90f9301b8dd4418c6544 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 26 May 2022 18:32:47 -0400 Subject: [PATCH 086/113] Rejig config helpers for arbitrary named files --- piker/config.py | 82 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/piker/config.py b/piker/config.py index cf946405a..d1926decb 100644 --- a/piker/config.py +++ b/piker/config.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) +# Copyright (C) 2018-present Tyler Goodlet (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,6 +16,7 @@ """ Broker configuration mgmt. + """ import platform import sys @@ -50,7 +51,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): Unix (POSIX): ``~/.foo-bar`` Win XP (roaming): - ``C:\Documents and Settings\\Local Settings\Application Data\Foo Bar`` + ``C:\Documents and Settings\\Local Settings\Application Data\Foo`` Win XP (not roaming): ``C:\Documents and Settings\\Application Data\Foo Bar`` Win 7 (roaming): @@ -81,7 +82,8 @@ def _posixify(name): folder = os.path.expanduser("~") return os.path.join(folder, app_name) if force_posix: - return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name)))) + return os.path.join( + os.path.expanduser("~/.{}".format(_posixify(app_name)))) if sys.platform == "darwin": return os.path.join( os.path.expanduser("~/Library/Application Support"), app_name @@ -107,7 +109,12 @@ def _posixify(name): ] ) -_file_name = 'brokers.toml' +_conf_names: set[str] = { + 'brokers', + 'trades', + 'watchlists', +} + _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json') _context_defaults = dict( default_map={ @@ -129,23 +136,43 @@ def _override_config_dir( _config_dir = path -def get_broker_conf_path(): +def _conf_fn_w_ext( + name: str, +) -> str: + # change this if we ever change the config file format. + return f'{name}.toml' + + +def get_conf_path( + conf_name: str = 'brokers', + +) -> str: """Return the default config path normally under ``~/.config/piker`` on linux. Contains files such as: - brokers.toml - watchlists.toml + - trades.toml + + # maybe coming soon ;) - signals.toml - strats.toml """ - return os.path.join(_config_dir, _file_name) + assert conf_name in _conf_names + fn = _conf_fn_w_ext(conf_name) + return os.path.join( + _config_dir, + fn, + ) def repodir(): - """Return the abspath to the repo directory. - """ + ''' + Return the abspath to the repo directory. + + ''' dirpath = os.path.abspath( # we're 3 levels down in **this** module file dirname(dirname(os.path.realpath(__file__))) @@ -154,16 +181,27 @@ def repodir(): def load( + conf_name: str = 'brokers', path: str = None + ) -> (dict, str): - """Load broker config. - """ - path = path or get_broker_conf_path() + ''' + Load config file by name. + + ''' + path = path or get_conf_path(conf_name) if not os.path.isfile(path): - shutil.copyfile( - os.path.join(repodir(), 'config', 'brokers.toml'), - path, + fn = _conf_fn_w_ext(conf_name) + + template = os.path.join( + repodir(), + 'config', + fn ) + # try to copy in a template config to the user's directory + # if one exists. + if os.path.isfile(template): + shutil.copyfile(template, path) config = toml.load(path) log.debug(f"Read config file {path}") @@ -172,13 +210,17 @@ def load( def write( config: dict, # toml config as dict + name: str = 'brokers', path: str = None, + ) -> None: - """Write broker config to disk. + '''' + Write broker config to disk. Create a ``brokers.ini`` file if one does not exist. - """ - path = path or get_broker_conf_path() + + ''' + path = path or get_conf_path(name) dirname = os.path.dirname(path) if not os.path.isdir(dirname): log.debug(f"Creating config dir {_config_dir}") @@ -188,7 +230,10 @@ def write( raise ValueError( "Watch out you're trying to write a blank config!") - log.debug(f"Writing config file {path}") + log.debug( + f"Writing config `{name}` file to:\n" + f"{path}" + ) with open(path, 'w') as cf: return toml.dump(config, cf) @@ -218,4 +263,5 @@ def load_accounts( # our default paper engine entry accounts['paper'] = None + return accounts From 88ac2fda52a882e60b0d7d211bf1bc9bfeb60afd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 28 May 2022 15:41:11 -0400 Subject: [PATCH 087/113] Aggretate cache resetting into a single ctx mngr method --- piker/ui/_curve.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index abe489295..119019879 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -18,6 +18,7 @@ Fast, smooth, sexy curves. """ +from contextlib import contextmanager as cm from typing import Optional import numpy as np @@ -38,7 +39,6 @@ # # ohlc_to_m4_line, # ds_m4, # ) -from ._pathops import xy_downsample from ..log import get_logger @@ -216,22 +216,11 @@ def clear(self): # self.fast_path.clear() self.fast_path = None - # self.disable_cache() - # self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - + @cm def reset_cache(self) -> None: - self.disable_cache() - self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) - - def disable_cache(self) -> None: - ''' - Disable the use of the pixel coordinate cache and trigger a geo event. - - ''' - # XXX: pretty annoying but, without this there's little - # artefacts on the append updates to the curve... self.setCacheMode(QtWidgets.QGraphicsItem.NoCache) - # self.prepareGeometryChange() + yield + self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) def boundingRect(self): ''' @@ -285,8 +274,8 @@ def _path_br(self): # actually terminates earlier so we don't need # this for the last step. w += self._last_w - ll = self._last_line - h += 1 #ll.y2() - ll.y1() + # ll = self._last_line + h += 1 # ll.y2() - ll.y1() # br = QPointF( # self._vr[-1], From d61b6364879f8c2ceae2cfd402c4c3ae06e3c26e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 29 May 2022 14:49:53 -0400 Subject: [PATCH 088/113] Auto-yrange overlays in interaction (downsampler) handler --- piker/ui/_interaction.py | 46 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 8e95855e9..aa2cefc2d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -747,9 +747,8 @@ def _set_yrange( # flag to prevent triggering sibling charts from the same linked # set from recursion errors. - autoscale_linked_plots: bool = True, + autoscale_linked_plots: bool = False, name: Optional[str] = None, - # autoscale_overlays: bool = False, ) -> None: ''' @@ -760,7 +759,7 @@ def _set_yrange( data set. ''' - # print(f'YRANGE ON {self.name}') + # log.info(f'YRANGE ON {self.name}') profiler = pg.debug.Profiler( msg=f'`ChartView._set_yrange()`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -795,7 +794,8 @@ def _set_yrange( # XXX: only compute the mxmn range # if none is provided as input! if not yrange: - yrange = self._maxmin() + yrange = self._maxmin( + ) if yrange is None: log.warning(f'No yrange provided for {self.name}!?') @@ -821,25 +821,6 @@ def _set_yrange( self.setYRange(ylow, yhigh) profiler(f'set limits: {(ylow, yhigh)}') - # TODO: maybe should be a method on the - # chart widget/item? - if autoscale_linked_plots: - # avoid recursion by sibling plots - linked = self.linkedsplits - plots = list(linked.subplots.copy().values()) - main = linked.chart - if main: - plots.append(main) - - # print(f'autoscaling linked: {plots}') - for chart in plots: - if chart and not chart._static_yrange: - chart.cv._set_yrange( - # bars_range=br, - autoscale_linked_plots=False, - ) - profiler('autoscaled linked plots') - profiler.finish() def enable_auto_yrange( @@ -911,7 +892,10 @@ def x_uppx(self) -> float: else: return 0 - def maybe_downsample_graphics(self): + def maybe_downsample_graphics( + self, + autoscale_linked_plots: bool = True, + ): profiler = pg.debug.Profiler( msg=f'ChartView.maybe_downsample_graphics() for {self.name}', @@ -945,9 +929,17 @@ def maybe_downsample_graphics(self): chart.update_graphics_from_flow( name, use_vr=True, - - # gets passed down into graphics obj - # profiler=profiler, ) + # for each overlay on this chart auto-scale the + # y-range to max-min values. + if autoscale_linked_plots: + overlay = chart.pi_overlay + if overlay: + for pi in overlay.overlays: + pi.vb._set_yrange( + # bars_range=br, + ) + profiler('autoscaled linked plots') + profiler(f'<{chart_name}>.update_graphics_from_flow({name})') From a9ec1a97dd54844dc8fc7de6a37360b71ce2780a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 29 May 2022 23:43:31 -0400 Subject: [PATCH 089/113] Vlm "rate" fsps, change maxmin callback name to include `multi_` --- piker/ui/_fsp.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/piker/ui/_fsp.py b/piker/ui/_fsp.py index 3d90f0140..af03a9c64 100644 --- a/piker/ui/_fsp.py +++ b/piker/ui/_fsp.py @@ -635,7 +635,7 @@ async def open_vlm_displays( ) # force 0 to always be in view - def maxmin( + def multi_maxmin( names: list[str], ) -> tuple[float, float]: @@ -651,7 +651,7 @@ def maxmin( return 0, mx - chart.view.maxmin = partial(maxmin, names=['volume']) + chart.view.maxmin = partial(multi_maxmin, names=['volume']) # TODO: fix the x-axis label issue where if you put # the axis on the left it's totally not lined up... @@ -741,19 +741,20 @@ def maxmin( 'dolla_vlm', 'dark_vlm', ] - dvlm_rate_fields = [ - 'dvlm_rate', - 'dark_dvlm_rate', - ] + # dvlm_rate_fields = [ + # 'dvlm_rate', + # 'dark_dvlm_rate', + # ] trade_rate_fields = [ 'trade_rate', 'dark_trade_rate', ] group_mxmn = partial( - maxmin, + multi_maxmin, # keep both regular and dark vlm in view - names=fields + dvlm_rate_fields, + names=fields, + # names=fields + dvlm_rate_fields, ) # add custom auto range handler @@ -820,11 +821,11 @@ def chart_curves( ) await started.wait() - chart_curves( - dvlm_rate_fields, - dvlm_pi, - fr_shm, - ) + # chart_curves( + # dvlm_rate_fields, + # dvlm_pi, + # fr_shm, + # ) # TODO: is there a way to "sync" the dual axes such that only # one curve is needed? @@ -862,7 +863,7 @@ def chart_curves( ) # add custom auto range handler tr_pi.vb.maxmin = partial( - maxmin, + multi_maxmin, # keep both regular and dark vlm in view names=trade_rate_fields, ) From ab0def22c1d126925b91eca11b1de0ea2eaa44b3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 09:26:41 -0400 Subject: [PATCH 090/113] Change flag name to `autoscale_overlays` --- piker/ui/_interaction.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index aa2cefc2d..9d7159c33 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -759,9 +759,10 @@ def _set_yrange( data set. ''' - # log.info(f'YRANGE ON {self.name}') + name = self.name + # print(f'YRANGE ON {name}') profiler = pg.debug.Profiler( - msg=f'`ChartView._set_yrange()`: `{self.name}`', + msg=f'`ChartView._set_yrange()`: `{name}`', disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, delayed=True, @@ -794,12 +795,12 @@ def _set_yrange( # XXX: only compute the mxmn range # if none is provided as input! if not yrange: - yrange = self._maxmin( - ) + # flow = chart._flows[name] + yrange = self._maxmin() if yrange is None: - log.warning(f'No yrange provided for {self.name}!?') - print(f"WTF NO YRANGE {self.name}") + log.warning(f'No yrange provided for {name}!?') + print(f"WTF NO YRANGE {name}") return ylow, yhigh = yrange @@ -894,7 +895,7 @@ def x_uppx(self) -> float: def maybe_downsample_graphics( self, - autoscale_linked_plots: bool = True, + autoscale_overlays: bool = True, ): profiler = pg.debug.Profiler( @@ -933,7 +934,7 @@ def maybe_downsample_graphics( # for each overlay on this chart auto-scale the # y-range to max-min values. - if autoscale_linked_plots: + if autoscale_overlays: overlay = chart.pi_overlay if overlay: for pi in overlay.overlays: From 360643b32f7770ba5c6927269c45bb45e1b51b23 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 09:37:33 -0400 Subject: [PATCH 091/113] Fix optional input `bars_range` type to match `Flow.datums_range()` --- piker/ui/_chart.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 31ca06041..329f15f59 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1251,7 +1251,9 @@ def in_view( def maxmin( self, name: Optional[str] = None, - bars_range: Optional[tuple[int, int, int, int]] = None, + bars_range: Optional[tuple[ + int, int, int, int, int, int + ]] = None, ) -> tuple[float, float]: ''' @@ -1260,6 +1262,7 @@ def maxmin( If ``bars_range`` is provided use that range. ''' + # print(f'Chart[{self.name}].maxmin()') profiler = pg.debug.Profiler( msg=f'`{str(self)}.maxmin(name={name})`: `{self.name}`', disabled=not pg_profile_enabled(), @@ -1279,7 +1282,14 @@ def maxmin( key = res = 0, 0 else: - first, l, lbar, rbar, r, last = bars_range or flow.datums_range() + ( + first, + l, + lbar, + rbar, + r, + last, + ) = bars_range or flow.datums_range() profiler(f'{self.name} got bars range') key = round(lbar), round(rbar) From 2c2c453932e4675f27265d25297a32183ca01007 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 19:02:06 -0400 Subject: [PATCH 092/113] Reset line graphics on downsample step.. This was a bit of a nightmare to figure out but, it seems that the coordinate caching system will really be a dick (like the nickname for richard for you serious types) about leaving stale graphics if we don't reset the cache on downsample full-redraw updates...Sooo, instead we do this manual reset to avoid such artifacts and consequently (for now) return a `reset: bool` flag in the return tuple from `Renderer.render()` to indicate as such. Some further shite: - move the step mode `.draw_last()` equivalent graphics updates down with the rest.. - drop some superfluous `should_redraw` logic from `Renderer.render()` and compound it in the full path redraw block. --- piker/ui/_flows.py | 105 ++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index e74ee123f..8610bb680 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -570,7 +570,6 @@ def update_graphics( draw_last: bool = True slice_to_head: int = -1 - # input_data = None should_redraw: bool = False @@ -616,7 +615,8 @@ def update_graphics( should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - if getattr(graphics, '_step_mode', False): + step_mode = getattr(graphics, '_step_mode', False) + if step_mode: r.allocate_xy = to_step_format r.update_xy = update_step_xy @@ -636,16 +636,7 @@ def update_graphics( x_last = x[-1] y_last = y[-1] - w = 0.5 - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - draw_last = False + draw_last = True # downsampling incremental state checking # check for and set std m4 downsample conditions @@ -660,10 +651,11 @@ def update_graphics( f'{array_key} sampler change: {self._last_uppx} -> {uppx}' ) self._last_uppx = uppx + new_sample_rate = True showing_src_data = False - should_redraw = True should_ds = True + should_redraw = True elif ( uppx <= 2 @@ -672,17 +664,17 @@ def update_graphics( # we should de-downsample back to our original # source data so we clear our path data in prep # to generate a new one from original source data. - should_redraw = True new_sample_rate = True - should_ds = False showing_src_data = True + should_ds = False + should_redraw = True # MAIN RENDER LOGIC: # - determine in view data and redraw on range change # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path, data = r.render( + path, data, reset = r.render( read, array_key, profiler, @@ -702,29 +694,49 @@ def update_graphics( **rkwargs, ) - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). - # graphics.prepareGeometryChange() - # assign output paths to graphicis obj - graphics.path = r.path - graphics.fast_path = r.fast_path + # XXX: SUPER UGGGHHH... without this we get stale cache + # graphics that don't update until you downsampler again.. + if reset: + with graphics.reset_cache(): + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path + else: + # assign output paths to graphicis obj + graphics.path = r.path + graphics.fast_path = r.fast_path if draw_last and not bars: - # TODO: how to handle this draw last stuff.. - x = data['index'] - y = data[array_key] - graphics.draw_last(x, y) + # default line draw last call + if not step_mode: + with graphics.reset_cache(): + x = data['index'] + y = data[array_key] + graphics.draw_last(x, y) - elif bars and draw_last: - draw_last() + else: + w = 0.5 + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) - profiler('draw last segment') + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.prepareGeometryChange()') - graphics.update() - profiler('.update()') + elif bars and draw_last: + draw_last() + graphics.update() + profiler('.update()') - profiler('`graphics.update_from_array()` complete') return graphics @@ -1009,8 +1021,8 @@ def render( if use_vr: array = in_view - else: - ivl, ivr = xfirst, xlast + # else: + # ivl, ivr = xfirst, xlast hist = array[:slice_to_head] @@ -1069,24 +1081,22 @@ def render( ): zoom_or_append = True - if ( - view_range != last_vr - and ( - append_length > 1 - or zoom_or_append - ) - ): - should_redraw = True - self._last_vr = view_range if len(x_out): self._last_ivr = x_out[0], x_out[slice_to_head] - if prepend_length > 0: + # redraw conditions + if ( + prepend_length > 0 + or new_sample_rate + or append_length > 0 + or zoom_or_append + ): should_redraw = True path = self.path fast_path = self.fast_path + reset = False # redraw the entire source data if we have either of: # - no prior path graphic rendered or, @@ -1094,10 +1104,8 @@ def render( if ( path is None or should_redraw - or new_sample_rate - or prepend_length > 0 ): - # print("REDRAWING BRUH") + # print(f"{self.flow.name} -> REDRAWING BRUH") if new_sample_rate and showing_src_data: log.info(f'DEDOWN -> {array_key}') self._in_ds = False @@ -1109,6 +1117,7 @@ def render( y_out, uppx, ) + reset = True profiler(f'FULL PATH downsample redraw={should_ds}') self._in_ds = True @@ -1203,4 +1212,4 @@ def render( # update diff state since we've now rendered paths. self.last_read = new_read - return self.path, array + return self.path, array, reset From 6f00617bd3a341a9cc15138c9cd124efa491973e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 30 May 2022 20:01:40 -0400 Subject: [PATCH 093/113] Only do new "datum append" when visible in pixels The basic logic is now this: - when zooming out, uppx (units per pixel in x) can be >= 1 - if the uppx is `n` then the next pixel in view becomes occupied by a new datum-x-coordinate-value when the diff between the last datum step (since the last such update) is greater then the current uppx -> `datums_diff >= n` - if we're less then some constant uppx we just always update (because it's not costly enough and we're not downsampling. More or less this just avoids unnecessary real-time updates to flow graphics until they would actually be noticeable via the next pixel column on screen. --- piker/ui/_display.py | 58 +++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index c551fc98c..e33c2c741 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -32,7 +32,7 @@ import pendulum import pyqtgraph as pg -from .. import brokers +# from .. import brokers from ..data.feed import open_feed from ._axes import YAxisLabel from ._chart import ( @@ -263,6 +263,7 @@ async def graphics_update_loop( 'vars': { 'tick_margin': tick_margin, 'i_last': i_last, + 'i_last_append': i_last, 'last_mx_vlm': last_mx_vlm, 'last_mx': last_mx, 'last_mn': last_mn, @@ -320,8 +321,8 @@ def graphics_update_cycle( profiler = pg.debug.Profiler( msg=f'Graphics loop cycle for: `{chart.name}`', delayed=True, - # disabled=not pg_profile_enabled(), - disabled=True, + disabled=not pg_profile_enabled(), + # disabled=True, ms_threshold=ms_slower_then, # ms_threshold=1/12 * 1e3, @@ -340,7 +341,7 @@ def graphics_update_cycle( for sym, quote in ds.quotes.items(): # compute the first available graphic's x-units-per-pixel - xpx = vlm_chart.view.x_uppx() + uppx = vlm_chart.view.x_uppx() # NOTE: vlm may be written by the ``brokerd`` backend # event though a tick sample is not emitted. @@ -359,13 +360,32 @@ def graphics_update_cycle( i_diff = i_step - vars['i_last'] vars['i_last'] = i_step + append_diff = i_step - vars['i_last_append'] + + # update the "last datum" (aka extending the flow graphic with + # new data) only if the number of unit steps is >= the number of + # such unit steps per pixel (aka uppx). Iow, if the zoom level + # is such that a datum(s) update to graphics wouldn't span + # to a new pixel, we don't update yet. + do_append = (append_diff >= uppx) + if do_append: + vars['i_last_append'] = i_step + + do_rt_update = uppx < update_uppx + # print( + # f'append_diff:{append_diff}\n' + # f'uppx:{uppx}\n' + # f'do_append: {do_append}' + # ) + + # TODO: we should only run mxmn when we know + # an update is due via ``do_append`` above. ( brange, mx_in_view, mn_in_view, mx_vlm_in_view, ) = ds.maxmin() - l, lbar, rbar, r = brange mx = mx_in_view + tick_margin mn = mn_in_view - tick_margin @@ -389,8 +409,9 @@ def graphics_update_cycle( # left unless we get one of the following: if ( ( - i_diff > 0 # no new sample step - and xpx < 4 # chart is zoomed out very far + # i_diff > 0 # no new sample step + do_append + # and uppx < 4 # chart is zoomed out very far and liv ) or trigger_all @@ -399,6 +420,10 @@ def graphics_update_cycle( # pixel in a curve should show new data based on uppx # and then iff update curves and shift? chart.increment_view(steps=i_diff) + + if vlm_chart: + vlm_chart.increment_view(steps=i_diff) + profiler('view incremented') if vlm_chart: @@ -409,8 +434,8 @@ def graphics_update_cycle( if ( ( - xpx < update_uppx - or i_diff > 0 + do_rt_update + or do_append and liv ) or trigger_all @@ -454,14 +479,15 @@ def graphics_update_cycle( flow, curve_name, array_key=curve_name, - do_append=xpx < update_uppx, + # do_append=uppx < update_uppx, + do_append=do_append, ) # is this even doing anything? # (pretty sure it's the real-time # resizing from last quote?) fvb = flow.plot.vb fvb._set_yrange( - autoscale_linked_plots=False, + # autoscale_linked_plots=False, name=curve_name, ) @@ -510,13 +536,17 @@ def graphics_update_cycle( # update ohlc sampled price bars if ( - xpx < update_uppx - or i_diff > 0 + do_rt_update + or do_append or trigger_all ): + # TODO: we should always update the "last" datum + # since the current range should at least be updated + # to it's max/min on the last pixel. chart.update_graphics_from_flow( chart.name, - do_append=xpx < update_uppx, + # do_append=uppx < update_uppx, + do_append=do_append, ) # iterate in FIFO order per tick-frame From 3ab91deaec5d0d59894b6dc63906f6d909fd26cc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 12:46:20 -0400 Subject: [PATCH 094/113] Drop all (old) unused state instance vars --- piker/ui/_curve.py | 13 ------------- piker/ui/_flows.py | 12 ++---------- piker/ui/_ohlc.py | 18 ------------------ 3 files changed, 2 insertions(+), 41 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 119019879..65e604281 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -84,8 +84,6 @@ def __init__( # brutaaalll, see comments within.. self.yData = None self.xData = None - # self._vr: Optional[tuple] = None - # self._avr: Optional[tuple] = None self._last_cap: int = 0 self._name = name @@ -98,14 +96,6 @@ def __init__( # we're basically only using the pen setting now... super().__init__(*args, **kwargs) - # self._xrange: tuple[int, int] = self.dataBounds(ax=0) - # self._xrange: Optional[tuple[int, int]] = None - # self._x_iv_range = None - - # self._last_draw = time.time() - # self._in_ds: bool = False - # self._last_uppx: float = 0 - # all history of curve is drawn in single px thickness pen = pg.mkPen(hcolor(color)) pen.setStyle(_line_styles[style]) @@ -162,9 +152,6 @@ def px_width(self) -> float: vr = self.viewRect() l, r = int(vr.left()), int(vr.right()) - # if not self._xrange: - # return 0 - start, stop = self._xrange lbar = max(l, start) rbar = min(r, stop) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 8610bb680..2086a3ece 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -358,20 +358,10 @@ class Flow(msgspec.Struct): # , frozen=True): is_ohlc: bool = False render: bool = True # toggle for display loop - # pre-graphics formatted data - gy: Optional[np.ndarray] = None - gx: Optional[np.ndarray] = None - - # pre-graphics update indices - _iflat_last: int = 0 - _iflat_first: int = 0 - # downsampling state _last_uppx: float = 0 _in_ds: bool = False - _graphics_tranform_fn: Optional[Callable[ShmArray, np.ndarray]] = None - # map from uppx -> (downsampled data, incremental graphics) _src_r: Optional[Renderer] = None _render_table: dict[ @@ -717,6 +707,8 @@ def update_graphics( else: w = 0.5 + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. graphics._last_line = QLineF( x_last - w, 0, x_last + w, 0, diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index f10a49985..e5542609e 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -129,11 +129,7 @@ def __init__( self.fast_path = QtGui.QPainterPath() self._xrange: tuple[int, int] - # self._yrange: tuple[float, float] self._vrange = None - - # TODO: don't render the full backing array each time - # self._path_data = None self._last_bar_lines: Optional[tuple[QLineF, ...]] = None # track the current length of drawable lines within the larger array @@ -212,16 +208,6 @@ def boundingRect(self): hb.bottomRight(), ) - # fp = self.fast_path - # if fp: - # fhb = fp.controlPointRect() - # print((hb_tl, hb_br)) - # print(fhb) - # hb_tl, hb_br = ( - # fhb.topLeft() + hb.topLeft(), - # fhb.bottomRight() + hb.bottomRight(), - # ) - # need to include last bar height or BR will be off mx_y = hb_br.y() mn_y = hb_tl.y() @@ -281,7 +267,3 @@ def paint( p.setPen(self.bars_pen) p.drawPath(self.path) profiler(f'draw history path: {self.path.capacity()}') - - # if self.fast_path: - # p.drawPath(self.fast_path) - # profiler('draw fast path') From 8f1faf97ee5ebe484ca3b79f94ac38de3f1e1656 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 12:46:50 -0400 Subject: [PATCH 095/113] Add todo for bars range reuse in interaction handler --- piker/ui/_interaction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 9d7159c33..a659612a2 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -939,6 +939,7 @@ def maybe_downsample_graphics( if overlay: for pi in overlay.overlays: pi.vb._set_yrange( + # TODO: get the range once up front... # bars_range=br, ) profiler('autoscaled linked plots') From 57acc3bd29417923cb4e376b6034c6a8941335b5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 13:57:10 -0400 Subject: [PATCH 096/113] Factor all per graphic `.draw_last()` methods into closures --- piker/ui/_curve.py | 17 ----- piker/ui/_flows.py | 173 ++++++++++++++++++++++++++++++++++----------- piker/ui/_ohlc.py | 40 ----------- 3 files changed, 133 insertions(+), 97 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 65e604281..89180200b 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -160,23 +160,6 @@ def px_width(self) -> float: QLineF(lbar, 0, rbar, 0) ).length() - def draw_last( - self, - x: np.ndarray, - y: np.ndarray, - - ) -> None: - x_last = x[-1] - y_last = y[-1] - - # draw the "current" step graphic segment so it lines up with - # the "middle" of the current (OHLC) sample. - self._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) - # self._last_w = x_last - x[-2] - # XXX: lol brutal, the internals of `CurvePoint` (inherited by # our `LineDot`) required ``.getData()`` to work.. def getData(self): diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 2086a3ece..907446a2a 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -23,7 +23,6 @@ ''' from __future__ import annotations -from functools import partial from typing import ( Optional, Callable, @@ -58,6 +57,7 @@ ) from ._ohlc import ( BarItems, + bar_from_ohlc_row, ) from ._curve import ( FastAppendCurve, @@ -245,18 +245,73 @@ def render_baritems( bars.update() draw_last = False - lasts = self.shm.array[-2:] - last = lasts[-1] if should_line: - def draw_last(): + + def draw_last_flattened_ohlc_line( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + ) -> None: + lasts = src_data[-2:] x, y = lasts['index'], lasts['close'] - curve.draw_last(x, y) + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + graphics._last_line = QLineF( + x[-2], y[-2], + x[-1], y[-1] + ) + + draw_last = draw_last_flattened_ohlc_line + else: - draw_last = partial( - bars.draw_last, - last, - ) + def draw_last_ohlc_bar( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + ) -> None: + last = src_data[-1] + # generate new lines objects for updatable "current bar" + graphics._last_bar_lines = bar_from_ohlc_row(last, graphics.w) + + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == graphics.start_index - 1 + # assert i == last_index + body, larm, rarm = graphics._last_bar_lines + + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) + + if l != h: # noqa + + if body is None: + body = graphics._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) + + # XXX: pretty sure this is causing an issue where the + # bar has a large upward move right before the next + # sample and the body is getting set to None since the + # next bar is flat but the shm array index update wasn't + # read by the time this code runs. Iow we're doing this + # removal of the body for a bar index that is now out of + # date / from some previous sample. It's weird though + # because i've seen it do this to bars i - 3 back? + + draw_last = draw_last_ohlc_bar return ( graphics, @@ -355,6 +410,12 @@ class Flow(msgspec.Struct): # , frozen=True): graphics: pg.GraphicsObject _shm: ShmArray + draw_last_datum: Optional[ + Callable[ + [np.ndarray, str], + tuple[np.ndarray] + ] + ] = None is_ohlc: bool = False render: bool = True # toggle for display loop @@ -543,7 +604,7 @@ def update_graphics( ) # shm read and slice to view read = ( - xfirst, xlast, array, + xfirst, xlast, src_array, ivl, ivr, in_view, ) = self.read() @@ -618,14 +679,6 @@ def update_graphics( # corrent yet for step curves.. remove this to see it. should_redraw = True - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = array['index'] - y = array[array_key] - x_last = x[-1] - y_last = y[-1] - draw_last = True # downsampling incremental state checking @@ -698,34 +751,74 @@ def update_graphics( graphics.fast_path = r.fast_path if draw_last and not bars: - # default line draw last call + if not step_mode: - with graphics.reset_cache(): - x = data['index'] - y = data[array_key] - graphics.draw_last(x, y) + + def draw_last_line( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + + ) -> None: + # default line draw last call + with graphics.reset_cache(): + x = render_data['index'] + y = render_data[array_key] + x_last = x[-1] + y_last = y[-1] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + graphics._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + + draw_last_line(graphics, path, src_array, data, reset) else: - w = 0.5 - # lol, commenting this makes step curves - # all "black" for me :eyeroll:.. - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). I think it avoid artifacts on - # wheel-scroll downsampling curve updates? - graphics.update() - profiler('.prepareGeometryChange()') + def draw_last_step( + graphics: pg.GraphicsObject, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + + ) -> None: + w = 0.5 + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = src_array['index'] + y = src_array[array_key] + x_last = x[-1] + y_last = y[-1] + + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. + graphics._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + graphics._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + draw_last_step(graphics, path, src_array, data, reset) + + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.prepareGeometryChange()') elif bars and draw_last: - draw_last() + draw_last(graphics, path, src_array, data, reset) graphics.update() profiler('.update()') diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index e5542609e..ad4495979 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -32,8 +32,6 @@ from ._style import hcolor from ..log import get_logger from ._curve import FastAppendCurve -from ._compression import ohlc_flatten -from ._pathops import gen_ohlc_qpath if TYPE_CHECKING: from ._chart import LinkedSplits @@ -148,44 +146,6 @@ def x_uppx(self) -> int: else: return 0 - def draw_last( - self, - last: np.ndarray, - - ) -> None: - # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last, self.w) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == self.start_index - 1 - # assert i == last_index - body, larm, rarm = self._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = self._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the bar has - # a large upward move right before the next sample and the body - # is getting set to None since the next bar is flat but the shm - # array index update wasn't read by the time this code runs. Iow - # we're doing this removal of the body for a bar index that is - # now out of date / from some previous sample. It's weird - # though because i've seen it do this to bars i - 3 back? - def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect From a7ff47158bb3730f1728d82ad46d5168122a310b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 18:07:22 -0400 Subject: [PATCH 097/113] Pass tsdb flag when db is up XD --- piker/data/feed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 561d063ba..c49ab0fe5 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -338,7 +338,7 @@ def iter_dts(start: datetime): log.debug(f'New datetime index:\n{pformat(dtrange)}') for end_dt in dtrange: - log.warning(f'Yielding next frame start {end_dt}') + log.info(f'Yielding next frame start {end_dt}') start = yield end_dt # if caller sends a new start date, reset to that @@ -722,6 +722,7 @@ async def manage_history( bfqsn, shm, last_tsdb_dt=last_tsdb_dt, + tsdb_is_up=True, storage=storage, ) ) From fc24f5efd1826100275a3c4d5208e4250674a0b2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 31 May 2022 18:07:51 -0400 Subject: [PATCH 098/113] Iterate 1s and 1m from tsdb series --- piker/data/marketstore.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index 43b156718..4d1c91ad1 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -639,12 +639,13 @@ async def tsdb_history_update( tsdb_arrays = await storage.read_ohlcv(fqsn) # hist diffing if tsdb_arrays: - onesec = tsdb_arrays[1] - - # these aren't currently used but can be referenced from - # within the embedded ipython shell below. - to_append = ohlcv[ohlcv['time'] > onesec['Epoch'][-1]] - to_prepend = ohlcv[ohlcv['time'] < onesec['Epoch'][0]] + for secs in (1, 60): + ts = tsdb_arrays.get(secs) + if ts is not None and len(ts): + # these aren't currently used but can be referenced from + # within the embedded ipython shell below. + to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]] + to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]] profiler('Finished db arrays diffs') From 363ba8f9ae527649a3bcf9d27ce10f96baaa47b5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:13:08 -0400 Subject: [PATCH 099/113] Only drop throttle feeds if channel disconnects? --- piker/data/_sampling.py | 45 +++++++++++++++++++++++++++++++---------- piker/data/feed.py | 13 +++++++++--- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 10dc43f63..fda93e21b 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -22,7 +22,7 @@ from __future__ import annotations from collections import Counter import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Union import tractor import trio @@ -32,6 +32,7 @@ if TYPE_CHECKING: from ._sharedmem import ShmArray + from .feed import _FeedsBus log = get_logger(__name__) @@ -219,7 +220,7 @@ async def iter_ohlc_periods( async def sample_and_broadcast( - bus: '_FeedsBus', # noqa + bus: _FeedsBus, # noqa shm: ShmArray, quote_stream: trio.abc.ReceiveChannel, brokername: str, @@ -298,7 +299,13 @@ async def sample_and_broadcast( # end up triggering backpressure which which will # eventually block this producer end of the feed and # thus other consumers still attached. - subs = bus._subscribers[broker_symbol.lower()] + subs: list[ + tuple[ + Union[tractor.MsgStream, trio.MemorySendChannel], + tractor.Context, + Optional[float], # tick throttle in Hz + ] + ] = bus._subscribers[broker_symbol.lower()] # NOTE: by default the broker backend doesn't append # it's own "name" into the fqsn schema (but maybe it @@ -307,7 +314,7 @@ async def sample_and_broadcast( bsym = f'{broker_symbol}.{brokername}' lags: int = 0 - for (stream, tick_throttle) in subs: + for (stream, ctx, tick_throttle) in subs: try: with trio.move_on_after(0.2) as cs: @@ -319,11 +326,11 @@ async def sample_and_broadcast( (bsym, quote) ) except trio.WouldBlock: - ctx = getattr(stream, '_ctx', None) + chan = ctx.chan if ctx: log.warning( f'Feed overrun {bus.brokername} ->' - f'{ctx.channel.uid} !!!' + f'{chan.uid} !!!' ) else: key = id(stream) @@ -333,11 +340,26 @@ async def sample_and_broadcast( f'feed @ {tick_throttle} Hz' ) if overruns[key] > 6: - log.warning( - f'Dropping consumer {stream}' - ) - await stream.aclose() - raise trio.BrokenResourceError + # TODO: should we check for the + # context being cancelled? this + # could happen but the + # channel-ipc-pipe is still up. + if not chan.connected(): + log.warning( + 'Dropping broken consumer:\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) + await stream.aclose() + raise trio.BrokenResourceError + else: + log.warning( + 'Feed getting overrun bro!\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) + continue + else: await stream.send( {bsym: quote} @@ -482,6 +504,7 @@ async def uniform_rate_send( # if the feed consumer goes down then drop # out of this rate limiter log.warning(f'{stream} closed') + await stream.aclose() return # reset send cycle state diff --git a/piker/data/feed.py b/piker/data/feed.py index c49ab0fe5..94c2f81d6 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -33,6 +33,7 @@ Generator, Awaitable, TYPE_CHECKING, + Union, ) import trio @@ -117,7 +118,13 @@ class Config: # https://github.com/samuelcolvin/pydantic/issues/2816 _subscribers: dict[ str, - list[tuple[tractor.MsgStream, Optional[float]]] + list[ + tuple[ + Union[tractor.MsgStream, trio.MemorySendChannel], + tractor.Context, + Optional[float], # tick throttle in Hz + ] + ] ] = {} async def start_task( @@ -1118,10 +1125,10 @@ async def open_feed_bus( recv, stream, ) - sub = (send, tick_throttle) + sub = (send, ctx, tick_throttle) else: - sub = (stream, tick_throttle) + sub = (stream, ctx, tick_throttle) subs = bus._subscribers[bfqsn] subs.append(sub) From 064d18539521f6f9f304cbcbdd72bee18b275ce0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:13:31 -0400 Subject: [PATCH 100/113] Drop pointless geo call from `.pain()` --- piker/ui/_curve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 89180200b..64922356b 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -282,7 +282,6 @@ def paint( disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) - self.prepareGeometryChange() if ( self._step_mode From b71e8c5e6d02a5cc7bee607677c6fddbc176b24a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 12:14:15 -0400 Subject: [PATCH 101/113] Guard against empty source history slice output --- piker/ui/_flows.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 907446a2a..e0cc21f8a 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -256,7 +256,8 @@ def draw_last_flattened_ohlc_line( reset: bool, ) -> None: lasts = src_data[-2:] - x, y = lasts['index'], lasts['close'] + x = lasts['index'] + y = lasts['close'] # draw the "current" step graphic segment so it # lines up with the "middle" of the current @@ -717,7 +718,7 @@ def update_graphics( # - determine downsampling ops if needed # - (incrementally) update ``QPainterPath`` - path, data, reset = r.render( + out = r.render( read, array_key, profiler, @@ -738,6 +739,12 @@ def update_graphics( **rkwargs, ) + if not out: + log.warning(f'{self.name} failed to render!?') + return graphics + + path, data, reset = out + # XXX: SUPER UGGGHHH... without this we get stale cache # graphics that don't update until you downsampler again.. if reset: @@ -974,7 +981,7 @@ def draw_path( # the target display(s) on the sys. # if no_path_yet: # graphics.path.reserve(int(500e3)) - path=path, # path re-use / reserving + # path=path, # path re-use / reserving ) # avoid mem allocs if possible @@ -1113,6 +1120,9 @@ def render( # xy-path data transform: convert source data to a format # able to be passed to a `QPainterPath` rendering routine. + if not len(hist): + return + x_out, y_out, connect = self.format_xy( self, # TODO: hist here should be the pre-sliced From e6d03ba97fe8e9914ba2bc2202a5ba82d101db40 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 14:42:39 -0400 Subject: [PATCH 102/113] Add missing f-str prefix --- piker/clearing/_ems.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index e00676f26..17f9be1a1 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -80,7 +80,9 @@ def check_lt(price: float) -> bool: return check_lt - raise ValueError('trigger: {trigger_price}, last: {known_last}') + raise ValueError( + f'trigger: {trigger_price}, last: {known_last}' + ) @dataclass From 80835d4e04fe86bda09c6dc458a4fce4080b5c8b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 1 Jun 2022 15:01:30 -0400 Subject: [PATCH 103/113] More detailed rt feed drop logging --- piker/data/_sampling.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index fda93e21b..77b15d7fd 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -336,7 +336,8 @@ async def sample_and_broadcast( key = id(stream) overruns[key] += 1 log.warning( - f'Feed overrun {bus.brokername} -> ' + f'Feed overrun {broker_symbol}' + '@{bus.brokername} -> ' f'feed @ {tick_throttle} Hz' ) if overruns[key] > 6: @@ -375,11 +376,12 @@ async def sample_and_broadcast( trio.ClosedResourceError, trio.EndOfChannel, ): - ctx = getattr(stream, '_ctx', None) + chan = ctx.chan if ctx: log.warning( - f'{ctx.chan.uid} dropped ' - '`brokerd`-quotes-feed connection' + 'Dropped `brokerd`-quotes-feed connection:\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' ) if tick_throttle: assert stream._closed @@ -392,7 +394,11 @@ async def sample_and_broadcast( try: subs.remove((stream, tick_throttle)) except ValueError: - log.error(f'{stream} was already removed from subs!?') + log.error( + f'Stream was already removed from subs!?\n' + f'{broker_symbol}:' + f'{ctx.cid}@{chan.uid}' + ) # TODO: a less naive throttler, here's some snippets: From 0f4bfcdf22218bc611627b52e05ac30e428016e5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:34:36 -0400 Subject: [PATCH 104/113] Drop global pg settings --- piker/ui/_exec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 7b69acef4..1d1a9c3d4 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -49,10 +49,6 @@ log = get_logger(__name__) # pyqtgraph global config -# might as well enable this for now? -pg.useOpenGL = True -pg.enableExperimental = True - # engage core tweaks that give us better response # latency then the average pg user _do_overrides() From 4138cef512de08e9957bc8c0b5638d5499ad505b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:35:01 -0400 Subject: [PATCH 105/113] Drop old state from `BarsItems` --- piker/ui/_ohlc.py | 41 +++-------------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index ad4495979..d4a930655 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -31,7 +31,6 @@ from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor from ..log import get_logger -from ._curve import FastAppendCurve if TYPE_CHECKING: from ._chart import LinkedSplits @@ -42,6 +41,7 @@ def bar_from_ohlc_row( row: np.ndarray, + # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 ) -> tuple[QLineF]: @@ -87,9 +87,6 @@ class BarItems(pg.GraphicsObject): ''' sigPlotChanged = QtCore.pyqtSignal(object) - # 0.5 is no overlap between arms, 1.0 is full overlap - w: float = 0.43 - def __init__( self, linked: LinkedSplits, @@ -109,42 +106,13 @@ def __init__( self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2) self._name = name - self._ds_line_xy: Optional[ - tuple[np.ndarray, np.ndarray] - ] = None - - # NOTE: this prevents redraws on mouse interaction which is - # a huge boon for avg interaction latency. - - # TODO: one question still remaining is if this makes trasform - # interactions slower (such as zooming) and if so maybe if/when - # we implement a "history" mode for the view we disable this in - # that mode? self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - self._pi = plotitem self.path = QtGui.QPainterPath() - self.fast_path = QtGui.QPainterPath() - - self._xrange: tuple[int, int] - self._vrange = None self._last_bar_lines: Optional[tuple[QLineF, ...]] = None - # track the current length of drawable lines within the larger array - self.start_index: int = 0 - self.stop_index: int = 0 - - # downsampler-line state - self._in_ds: bool = False - self._ds_line: Optional[FastAppendCurve] = None - self._dsi: tuple[int, int] = 0, 0 - self._xs_in_px: float = 0 - def x_uppx(self) -> int: - if self._ds_line: - return self._ds_line.x_uppx() - else: - return 0 + # we expect the downsample curve report this. + return 0 def boundingRect(self): # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect @@ -203,9 +171,6 @@ def paint( ) -> None: - if self._in_ds: - return - profiler = pg.debug.Profiler( disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, From c518553aa9196abd09876b7e9f7f3d84c93985ec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:36:55 -0400 Subject: [PATCH 106/113] Add new curve doc string --- piker/ui/_curve.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 64922356b..0d32749e1 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -55,15 +55,28 @@ class FastAppendCurve(pg.GraphicsObject): ''' - A faster, append friendly version of ``pyqtgraph.PlotCurveItem`` - built for real-time data updates. - - The main difference is avoiding regeneration of the entire - historical path where possible and instead only updating the "new" - segment(s) via a ``numpy`` array diff calc. Further the "last" - graphic segment is drawn independently such that near-term (high - frequency) discrete-time-sampled style updates don't trigger a full - path redraw. + A faster, simpler, append friendly version of + ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time + updates. + + This type is a much stripped down version of a ``pyqtgraph`` style "graphics object" in + the sense that the internal lower level graphics which are drawn in the ``.paint()`` method + are actually rendered outside of this class entirely and instead are assigned as state + (instance vars) here and then drawn during a Qt graphics cycle. + + The main motivation for this more modular, composed design is that + lower level graphics data can be rendered in different threads and + then read and drawn in this main thread without having to worry + about dealing with Qt's concurrency primitives. See + ``piker.ui._flows.Renderer`` for details and logic related to lower + level path generation and incremental update. The main differences in + the path generation code include: + + - avoiding regeneration of the entire historical path where possible and instead + only updating the "new" segment(s) via a ``numpy`` array diff calc. + - here, the "last" graphics datum-segment is drawn independently + such that near-term (high frequency) discrete-time-sampled style + updates don't trigger a full path redraw. ''' def __init__( @@ -89,6 +102,9 @@ def __init__( self._name = name self.path: Optional[QtGui.QPainterPath] = None + # additional path used for appends which tries to avoid + # triggering an update/redraw of the presumably larger + # historical ``.path`` above. self.use_fpath = use_fpath self.fast_path: Optional[QtGui.QPainterPath] = None @@ -119,11 +135,13 @@ def __init__( # self._fill = True self._brush = pg.functions.mkBrush(hcolor(fill_color or color)) + # NOTE: this setting seems to mostly prevent redraws on mouse + # interaction which is a huge boon for avg interaction latency. + # TODO: one question still remaining is if this makes trasform # interactions slower (such as zooming) and if so maybe if/when # we implement a "history" mode for the view we disable this in # that mode? - # if step_mode: # don't enable caching by default for the case where the # only thing drawn is the "last" line segment which can # have a weird artifact where it won't be fully drawn to its From d770867163b1e240ce24472d0545cc9ac25b3dec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 13:38:14 -0400 Subject: [PATCH 107/113] Drop width arg to bar lines factory --- piker/ui/_flows.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index e0cc21f8a..c5e813ea7 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -254,6 +254,7 @@ def draw_last_flattened_ohlc_line( src_data: np.ndarray, render_data: np.ndarray, reset: bool, + ) -> None: lasts = src_data[-2:] x = lasts['index'] @@ -276,10 +277,12 @@ def draw_last_ohlc_bar( src_data: np.ndarray, render_data: np.ndarray, reset: bool, + ) -> None: last = src_data[-1] + # generate new lines objects for updatable "current bar" - graphics._last_bar_lines = bar_from_ohlc_row(last, graphics.w) + graphics._last_bar_lines = bar_from_ohlc_row(last) # last bar update i, o, h, l, last, v = last[ From 736178adfd78f8bc5fe5ed1bfea0583e27c7324f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jun 2022 18:11:59 -0400 Subject: [PATCH 108/113] Rename `FastAppendCurve` -> `Curve` --- piker/ui/_chart.py | 4 ++-- piker/ui/_curve.py | 4 ++-- piker/ui/_flows.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 329f15f59..bbab5a41b 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -50,7 +50,7 @@ from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems -from ._curve import FastAppendCurve +from ._curve import Curve from ._style import ( hcolor, CHART_MARGINS, @@ -1069,7 +1069,7 @@ def draw_curve( # yah, we wrote our own B) data = shm.array - curve = FastAppendCurve( + curve = Curve( # antialias=True, name=name, diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 0d32749e1..2cf9f0b78 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -53,7 +53,7 @@ } -class FastAppendCurve(pg.GraphicsObject): +class Curve(pg.GraphicsObject): ''' A faster, simpler, append friendly version of ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time @@ -296,7 +296,7 @@ def paint( ) -> None: profiler = pg.debug.Profiler( - msg=f'FastAppendCurve.paint(): `{self._name}`', + msg=f'Curve.paint(): `{self._name}`', disabled=not pg_profile_enabled(), ms_threshold=ms_slower_then, ) diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c5e813ea7..c2e6ec094 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -60,7 +60,7 @@ bar_from_ohlc_row, ) from ._curve import ( - FastAppendCurve, + Curve, ) from ..log import get_logger @@ -175,7 +175,7 @@ def render_baritems( format_xy=ohlc_flat_to_xy, ) - curve = FastAppendCurve( + curve = Curve( name=f'{flow.name}_ds_ohlc', color=bars._color, ) @@ -661,7 +661,7 @@ def update_graphics( last_read=read, ) - # ``FastAppendCurve`` case: + # ``Curve`` case: array_key = array_key or self.name # print(array_key) From 55772efb34b01ab633edd8e9d9771ae010ad5748 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 10:18:32 -0400 Subject: [PATCH 109/113] Bleh, try avoiding the too many files bug-thing.. --- piker/data/_sharedmem.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 47d58d3e6..1172fc7b3 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -20,6 +20,7 @@ """ from __future__ import annotations from sys import byteorder +import time from typing import Optional from multiprocessing.shared_memory import SharedMemory, _USE_POSIX @@ -546,7 +547,21 @@ def attach_shm_array( # https://stackoverflow.com/a/11103289 # attach to array buffer and view as per dtype - shm = SharedMemory(name=key) + _err: Optional[Exception] = None + for _ in range(3): + try: + shm = SharedMemory( + name=key, + create=False, + ) + break + except OSError as oserr: + _err = oserr + time.sleep(0.1) + else: + if _err: + raise _err + shmarr = np.ndarray( (size,), dtype=token.dtype, From a66934a49d7eea6d368eb175ceee38a05f8cb6f6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 13:55:34 -0400 Subject: [PATCH 110/113] Add `Curve` sub-types with new custom graphics API Instead of using a bunch of internal logic to modify low level paint-able elements create a `Curve` lineage that allows for graphics "style" customization via a small set of public methods: - `Curve.declare_paintables()` to allow setup of state/elements to be drawn in later methods. - `.sub_paint()` to allow painting additional elements along with the defaults. - `.sub_br()` to customize the `.boundingRect()` dimensions. - `.draw_last_datum()` which is expected to produce the paintable elements which will show the last datum in view. Introduce the new sub-types and load as necessary in `ChartPlotWidget.draw_curve()`: - `FlattenedOHLC` - `StepCurve` Reimplement all `.draw_last()` routines as a `Curve` method and call it the same way from `Flow.update_graphics()` --- piker/ui/_chart.py | 33 +++---- piker/ui/_curve.py | 212 +++++++++++++++++++++++++++++++++++++-------- piker/ui/_flows.py | 198 ++++++------------------------------------ piker/ui/_ohlc.py | 50 ++++++++++- 4 files changed, 264 insertions(+), 229 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index bbab5a41b..7b40f0d7c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -50,7 +50,10 @@ from ..data._sharedmem import ShmArray from ._l1 import L1Labels from ._ohlc import BarItems -from ._curve import Curve +from ._curve import ( + Curve, + StepCurve, +) from ._style import ( hcolor, CHART_MARGINS, @@ -1051,6 +1054,7 @@ def draw_curve( color: Optional[str] = None, add_label: bool = True, pi: Optional[pg.PlotItem] = None, + step_mode: bool = False, **pdi_kwargs, @@ -1067,29 +1071,18 @@ def draw_curve( data_key = array_key or name - # yah, we wrote our own B) - data = shm.array - curve = Curve( - # antialias=True, - name=name, - - # XXX: pretty sure this is just more overhead - # on data reads and makes graphics rendering no faster - # clipToView=True, + curve_type = { + None: Curve, + 'step': StepCurve, + # TODO: + # 'bars': BarsItems + }['step' if step_mode else None] + curve = curve_type( + name=name, **pdi_kwargs, ) - # XXX: see explanation for different caching modes: - # https://stackoverflow.com/a/39410081 - # seems to only be useful if we don't re-generate the entire - # QPainterPath every time - # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - - # don't ever use this - it's a colossal nightmare of artefacts - # and is disastrous for performance. - # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) - pi = pi or self.plotItem self._flows[data_key] = Flow( diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 2cf9f0b78..8feb24b98 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -19,20 +19,24 @@ """ from contextlib import contextmanager as cm -from typing import Optional +from typing import Optional, Callable import numpy as np import pyqtgraph as pg -from PyQt5 import QtGui, QtWidgets +from PyQt5 import QtWidgets from PyQt5.QtWidgets import QGraphicsItem from PyQt5.QtCore import ( Qt, QLineF, QSizeF, QRectF, + # QRect, QPointF, ) - +from PyQt5.QtGui import ( + QPainter, + QPainterPath, +) from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor # from ._compression import ( @@ -59,10 +63,12 @@ class Curve(pg.GraphicsObject): ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time updates. - This type is a much stripped down version of a ``pyqtgraph`` style "graphics object" in - the sense that the internal lower level graphics which are drawn in the ``.paint()`` method - are actually rendered outside of this class entirely and instead are assigned as state - (instance vars) here and then drawn during a Qt graphics cycle. + This type is a much stripped down version of a ``pyqtgraph`` style + "graphics object" in the sense that the internal lower level + graphics which are drawn in the ``.paint()`` method are actually + rendered outside of this class entirely and instead are assigned as + state (instance vars) here and then drawn during a Qt graphics + cycle. The main motivation for this more modular, composed design is that lower level graphics data can be rendered in different threads and @@ -72,13 +78,20 @@ class Curve(pg.GraphicsObject): level path generation and incremental update. The main differences in the path generation code include: - - avoiding regeneration of the entire historical path where possible and instead - only updating the "new" segment(s) via a ``numpy`` array diff calc. + - avoiding regeneration of the entire historical path where possible + and instead only updating the "new" segment(s) via a ``numpy`` + array diff calc. - here, the "last" graphics datum-segment is drawn independently such that near-term (high frequency) discrete-time-sampled style updates don't trigger a full path redraw. ''' + + # sub-type customization methods + sub_br: Optional[Callable] = None + sub_paint: Optional[Callable] = None + declare_paintables: Optional[Callable] = None + def __init__( self, *args, @@ -94,19 +107,20 @@ def __init__( ) -> None: + self._name = name + # brutaaalll, see comments within.. self.yData = None self.xData = None - self._last_cap: int = 0 - self._name = name - self.path: Optional[QtGui.QPainterPath] = None + # self._last_cap: int = 0 + self.path: Optional[QPainterPath] = None # additional path used for appends which tries to avoid # triggering an update/redraw of the presumably larger # historical ``.path`` above. self.use_fpath = use_fpath - self.fast_path: Optional[QtGui.QPainterPath] = None + self.fast_path: Optional[QPainterPath] = None # TODO: we can probably just dispense with the parent since # we're basically only using the pen setting now... @@ -125,12 +139,12 @@ def __init__( # self.last_step_pen = pg.mkPen(hcolor(color), width=2) self.last_step_pen = pg.mkPen(pen, width=2) - self._last_line: Optional[QLineF] = None - self._last_step_rect: Optional[QRectF] = None + # self._last_line: Optional[QLineF] = None + self._last_line = QLineF() self._last_w: float = 1 # flat-top style histogram-like discrete curve - self._step_mode: bool = step_mode + # self._step_mode: bool = step_mode # self._fill = True self._brush = pg.functions.mkBrush(hcolor(fill_color or color)) @@ -148,6 +162,21 @@ def __init__( # endpoint (something we saw on trade rate curves) self.setCacheMode(QGraphicsItem.DeviceCoordinateCache) + # XXX: see explanation for different caching modes: + # https://stackoverflow.com/a/39410081 + # seems to only be useful if we don't re-generate the entire + # QPainterPath every time + # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) + + # don't ever use this - it's a colossal nightmare of artefacts + # and is disastrous for performance. + # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache) + + # allow sub-type customization + declare = self.declare_paintables + if declare: + declare() + # TODO: probably stick this in a new parent # type which will contain our own version of # what ``PlotCurveItem`` had in terms of base @@ -215,7 +244,7 @@ def boundingRect(self): Compute and then cache our rect. ''' if self.path is None: - return QtGui.QPainterPath().boundingRect() + return QPainterPath().boundingRect() else: # dynamically override this method after initial # path is created to avoid requiring the above None check @@ -227,14 +256,15 @@ def _path_br(self): Post init ``.boundingRect()```. ''' - hb = self.path.controlPointRect() # hb = self.path.boundingRect() + hb = self.path.controlPointRect() hb_size = hb.size() fp = self.fast_path if fp: fhb = fp.controlPointRect() hb_size = fhb.size() + hb_size + # print(f'hb_size: {hb_size}') # if self._last_step_rect: @@ -255,7 +285,13 @@ def _path_br(self): w = hb_size.width() h = hb_size.height() - if not self._last_step_rect: + sbr = self.sub_br + if sbr: + w, h = self.sub_br(w, h) + else: + # assume plain line graphic and use + # default unit step in each direction. + # only on a plane line do we include # and extra index step's worth of width # since in the step case the end of the curve @@ -289,7 +325,7 @@ def _path_br(self): def paint( self, - p: QtGui.QPainter, + p: QPainter, opt: QtWidgets.QStyleOptionGraphicsItem, w: QtWidgets.QWidget @@ -301,25 +337,16 @@ def paint( ms_threshold=ms_slower_then, ) - if ( - self._step_mode - and self._last_step_rect - ): - brush = self._brush + sub_paint = self.sub_paint + if sub_paint: + sub_paint(p, profiler) - # p.drawLines(*tuple(filter(bool, self._last_step_lines))) - # p.drawRect(self._last_step_rect) - p.fillRect(self._last_step_rect, brush) - profiler('.fillRect()') - - if self._last_line: - p.setPen(self.last_step_pen) - p.drawLine(self._last_line) - profiler('.drawLine()') - p.setPen(self._pen) + p.setPen(self.last_step_pen) + p.drawLine(self._last_line) + profiler('.drawLine()') + p.setPen(self._pen) path = self.path - # cap = path.capacity() # if cap != self._last_cap: # print(f'NEW CAPACITY: {self._last_cap} -> {cap}') @@ -341,3 +368,116 @@ def paint( # if self._fill: # brush = self.opts['brush'] # p.fillPath(self.path, brush) + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + # default line draw last call + with self.reset_cache(): + x = render_data['index'] + y = render_data[array_key] + + x_last = x[-1] + y_last = y[-1] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + self._last_line = QLineF( + x[-2], y[-2], + x_last, y_last + ) + + +# TODO: this should probably be a "downsampled" curve type +# that draws a bar-style (but for the px column) last graphics +# element such that the current datum in view can be shown +# (via it's max / min) even when highly zoomed out. +class FlattenedOHLC(Curve): + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + lasts = src_data[-2:] + x = lasts['index'] + y = lasts['close'] + + # draw the "current" step graphic segment so it + # lines up with the "middle" of the current + # (OHLC) sample. + self._last_line = QLineF( + x[-2], y[-2], + x[-1], y[-1] + ) + + +class StepCurve(Curve): + + def declare_paintables( + self, + ) -> None: + self._last_step_rect = QRectF() + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + w: float = 0.5, + + ) -> None: + + # TODO: remove this and instead place all step curve + # updating into pre-path data render callbacks. + # full input data + x = src_data['index'] + y = src_data[array_key] + + x_last = x[-1] + y_last = y[-1] + + # lol, commenting this makes step curves + # all "black" for me :eyeroll:.. + self._last_line = QLineF( + x_last - w, 0, + x_last + w, 0, + ) + self._last_step_rect = QRectF( + x_last - w, 0, + x_last + w, y_last, + ) + + def sub_paint( + self, + p: QPainter, + profiler: pg.debug.Profiler, + + ) -> None: + # p.drawLines(*tuple(filter(bool, self._last_step_lines))) + # p.drawRect(self._last_step_rect) + p.fillRect(self._last_step_rect, self._brush) + profiler('.fillRect()') + + def sub_br( + self, + path_w: float, + path_h: float, + + ) -> (float, float): + # passthrough + return path_w, path_h diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index c2e6ec094..7960d6499 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,13 +34,6 @@ from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath -from PyQt5.QtCore import ( - # Qt, - QLineF, - # QSizeF, - QRectF, - # QPointF, -) from ..data._sharedmem import ( ShmArray, @@ -57,10 +50,12 @@ ) from ._ohlc import ( BarItems, - bar_from_ohlc_row, + # bar_from_ohlc_row, ) from ._curve import ( Curve, + StepCurve, + FlattenedOHLC, ) from ..log import get_logger @@ -175,7 +170,7 @@ def render_baritems( format_xy=ohlc_flat_to_xy, ) - curve = Curve( + curve = FlattenedOHLC( name=f'{flow.name}_ds_ohlc', color=bars._color, ) @@ -244,84 +239,10 @@ def render_baritems( bars.show() bars.update() - draw_last = False - - if should_line: - - def draw_last_flattened_ohlc_line( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - lasts = src_data[-2:] - x = lasts['index'] - y = lasts['close'] - - # draw the "current" step graphic segment so it - # lines up with the "middle" of the current - # (OHLC) sample. - graphics._last_line = QLineF( - x[-2], y[-2], - x[-1], y[-1] - ) - - draw_last = draw_last_flattened_ohlc_line - - else: - def draw_last_ohlc_bar( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - last = src_data[-1] - - # generate new lines objects for updatable "current bar" - graphics._last_bar_lines = bar_from_ohlc_row(last) - - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] - # assert i == graphics.start_index - 1 - # assert i == last_index - body, larm, rarm = graphics._last_bar_lines - - # XXX: is there a faster way to modify this? - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # writer is responsible for changing open on "first" volume of bar - larm.setLine(larm.x1(), o, larm.x2(), o) - - if l != h: # noqa - - if body is None: - body = graphics._last_bar_lines[0] = QLineF(i, l, i, h) - else: - # update body - body.setLine(i, l, i, h) - - # XXX: pretty sure this is causing an issue where the - # bar has a large upward move right before the next - # sample and the body is getting set to None since the - # next bar is flat but the shm array index update wasn't - # read by the time this code runs. Iow we're doing this - # removal of the body for a bar index that is now out of - # date / from some previous sample. It's weird though - # because i've seen it do this to bars i - 3 back? - - draw_last = draw_last_ohlc_bar - return ( graphics, r, {'read_from_key': False}, - draw_last, should_line, changed_to_line, ) @@ -411,10 +332,10 @@ class Flow(msgspec.Struct): # , frozen=True): ''' name: str plot: pg.PlotItem - graphics: pg.GraphicsObject + graphics: Curve _shm: ShmArray - draw_last_datum: Optional[ + draw_last: Optional[ Callable[ [np.ndarray, str], tuple[np.ndarray] @@ -597,12 +518,9 @@ def update_graphics( render to graphics. ''' - - # profiler = profiler or pg.debug.Profiler( profiler = pg.debug.Profiler( msg=f'Flow.update_graphics() for {self.name}', disabled=not pg_profile_enabled(), - # disabled=False, ms_threshold=4, # ms_threshold=ms_slower_then, ) @@ -623,13 +541,9 @@ def update_graphics( # print('exiting early') return graphics - draw_last: bool = True slice_to_head: int = -1 - should_redraw: bool = False - rkwargs = {} - bars = False if isinstance(graphics, BarItems): # XXX: special case where we change out graphics @@ -638,7 +552,6 @@ def update_graphics( graphics, r, rkwargs, - draw_last, should_line, changed_to_line, ) = render_baritems( @@ -648,7 +561,7 @@ def update_graphics( profiler, **kwargs, ) - bars = True + # bars = True should_redraw = changed_to_line or not should_line else: @@ -661,7 +574,7 @@ def update_graphics( last_read=read, ) - # ``Curve`` case: + # ``Curve`` derivative case(s): array_key = array_key or self.name # print(array_key) @@ -670,20 +583,19 @@ def update_graphics( should_ds: bool = r._in_ds showing_src_data: bool = not r._in_ds - step_mode = getattr(graphics, '_step_mode', False) + # step_mode = getattr(graphics, '_step_mode', False) + step_mode = isinstance(graphics, StepCurve) if step_mode: r.allocate_xy = to_step_format r.update_xy = update_step_xy r.format_xy = step_to_xy - slice_to_head = -2 - # TODO: append logic inside ``.render()`` isn't - # corrent yet for step curves.. remove this to see it. + # correct yet for step curves.. remove this to see it. should_redraw = True - - draw_last = True + # draw_last = True + slice_to_head = -2 # downsampling incremental state checking # check for and set std m4 downsample conditions @@ -760,77 +672,23 @@ def update_graphics( graphics.path = r.path graphics.fast_path = r.fast_path - if draw_last and not bars: - - if not step_mode: - - def draw_last_line( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - # default line draw last call - with graphics.reset_cache(): - x = render_data['index'] - y = render_data[array_key] - x_last = x[-1] - y_last = y[-1] - - # draw the "current" step graphic segment so it - # lines up with the "middle" of the current - # (OHLC) sample. - graphics._last_line = QLineF( - x[-2], y[-2], - x_last, y_last - ) - - draw_last_line(graphics, path, src_array, data, reset) + graphics.draw_last_datum( + path, + src_array, + data, + reset, + array_key, + ) - else: + # TODO: is this ever better? + # graphics.prepareGeometryChange() + # profiler('.prepareGeometryChange()') - def draw_last_step( - graphics: pg.GraphicsObject, - path: QPainterPath, - src_data: np.ndarray, - render_data: np.ndarray, - reset: bool, - - ) -> None: - w = 0.5 - # TODO: remove this and instead place all step curve - # updating into pre-path data render callbacks. - # full input data - x = src_array['index'] - y = src_array[array_key] - x_last = x[-1] - y_last = y[-1] - - # lol, commenting this makes step curves - # all "black" for me :eyeroll:.. - graphics._last_line = QLineF( - x_last - w, 0, - x_last + w, 0, - ) - graphics._last_step_rect = QRectF( - x_last - w, 0, - x_last + w, y_last, - ) - - draw_last_step(graphics, path, src_array, data, reset) - - # TODO: does this actuallly help us in any way (prolly should - # look at the source / ask ogi). I think it avoid artifacts on - # wheel-scroll downsampling curve updates? - graphics.update() - profiler('.prepareGeometryChange()') - - elif bars and draw_last: - draw_last(graphics, path, src_array, data, reset) - graphics.update() - profiler('.update()') + # TODO: does this actuallly help us in any way (prolly should + # look at the source / ask ogi). I think it avoid artifacts on + # wheel-scroll downsampling curve updates? + graphics.update() + profiler('.update()') return graphics diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index d4a930655..0f7ce6f70 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -27,6 +27,7 @@ import pyqtgraph as pg from PyQt5 import QtCore, QtGui, QtWidgets from PyQt5.QtCore import QLineF, QPointF +from PyQt5.QtGui import QPainterPath from .._profile import pg_profile_enabled, ms_slower_then from ._style import hcolor @@ -85,8 +86,6 @@ class BarItems(pg.GraphicsObject): "Price range" bars graphics rendered from a OHLC sampled sequence. ''' - sigPlotChanged = QtCore.pyqtSignal(object) - def __init__( self, linked: LinkedSplits, @@ -107,7 +106,7 @@ def __init__( self._name = name self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache) - self.path = QtGui.QPainterPath() + self.path = QPainterPath() self._last_bar_lines: Optional[tuple[QLineF, ...]] = None def x_uppx(self) -> int: @@ -192,3 +191,48 @@ def paint( p.setPen(self.bars_pen) p.drawPath(self.path) profiler(f'draw history path: {self.path.capacity()}') + + def draw_last_datum( + self, + path: QPainterPath, + src_data: np.ndarray, + render_data: np.ndarray, + reset: bool, + array_key: str, + + ) -> None: + last = src_data[-1] + + # generate new lines objects for updatable "current bar" + self._last_bar_lines = bar_from_ohlc_row(last) + + # last bar update + i, o, h, l, last, v = last[ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + # assert i == graphics.start_index - 1 + # assert i == last_index + body, larm, rarm = self._last_bar_lines + + # XXX: is there a faster way to modify this? + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) + + if l != h: # noqa + + if body is None: + body = self._last_bar_lines[0] = QLineF(i, l, i, h) + else: + # update body + body.setLine(i, l, i, h) + + # XXX: pretty sure this is causing an issue where the + # bar has a large upward move right before the next + # sample and the body is getting set to None since the + # next bar is flat but the shm array index update wasn't + # read by the time this code runs. Iow we're doing this + # removal of the body for a bar index that is now out of + # date / from some previous sample. It's weird though + # because i've seen it do this to bars i - 3 back? From e5f96391e30a20835bcbd23402cf5fdd2833c46a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 16:44:04 -0400 Subject: [PATCH 111/113] Return xy data from `Curve.draw_last_datum()` methods --- piker/ui/_curve.py | 9 +++++---- piker/ui/_ohlc.py | 24 ++++++++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/piker/ui/_curve.py b/piker/ui/_curve.py index 8feb24b98..ac967bf7b 100644 --- a/piker/ui/_curve.py +++ b/piker/ui/_curve.py @@ -383,17 +383,16 @@ def draw_last_datum( x = render_data['index'] y = render_data[array_key] - x_last = x[-1] - y_last = y[-1] - # draw the "current" step graphic segment so it # lines up with the "middle" of the current # (OHLC) sample. self._last_line = QLineF( x[-2], y[-2], - x_last, y_last + x[-1], y[-1], ) + return x, y + # TODO: this should probably be a "downsampled" curve type # that draws a bar-style (but for the px column) last graphics @@ -421,6 +420,7 @@ def draw_last_datum( x[-2], y[-2], x[-1], y[-1] ) + return x, y class StepCurve(Curve): @@ -461,6 +461,7 @@ def draw_last_datum( x_last - w, 0, x_last + w, y_last, ) + return x, y def sub_paint( self, diff --git a/piker/ui/_ohlc.py b/piker/ui/_ohlc.py index 0f7ce6f70..dbe4c18e7 100644 --- a/piker/ui/_ohlc.py +++ b/piker/ui/_ohlc.py @@ -200,16 +200,26 @@ def draw_last_datum( reset: bool, array_key: str, + fields: list[str] = [ + 'index', + 'open', + 'high', + 'low', + 'close', + ], + ) -> None: - last = src_data[-1] + + # relevant fields + ohlc = src_data[fields] + last_row = ohlc[-1:] + + # individual values + last_row = i, o, h, l, last = ohlc[-1] # generate new lines objects for updatable "current bar" - self._last_bar_lines = bar_from_ohlc_row(last) + self._last_bar_lines = bar_from_ohlc_row(last_row) - # last bar update - i, o, h, l, last, v = last[ - ['index', 'open', 'high', 'low', 'close', 'volume'] - ] # assert i == graphics.start_index - 1 # assert i == last_index body, larm, rarm = self._last_bar_lines @@ -236,3 +246,5 @@ def draw_last_datum( # removal of the body for a bar index that is now out of # date / from some previous sample. It's weird though # because i've seen it do this to bars i - 3 back? + + return ohlc['index'], ohlc['close'] From 99965e7601a2e5a7522e7a7086128ed29d1c122f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jun 2022 16:45:53 -0400 Subject: [PATCH 112/113] Only draw mx/mn line for last uppx's worth of datums When using m4, we downsample to the max and min of each pixel-column's-worth of data thus preserving range / dispersion details whilst not drawing more graphics then can be displayed by the available amount of horizontal pixels. Take and apply this exact same concept to the "last datum" graphics elements for any `Flow` that is reported as being in a downsampled state: - take the xy output from the `Curve.draw_last_datum()`, - slice out all data that fits in the last pixel's worth of x-range by using the uppx, - compute the highest and lowest value from that data, - draw a singe line segment which spans this yrange thus creating a simple vertical set of pixels which are "filled in" and show the entire y-range for the most recent data "contained with that pixel". --- piker/ui/_display.py | 13 ++++++++++++ piker/ui/_flows.py | 47 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/piker/ui/_display.py b/piker/ui/_display.py index e33c2c741..415827fb7 100644 --- a/piker/ui/_display.py +++ b/piker/ui/_display.py @@ -654,6 +654,19 @@ def graphics_update_cycle( # run synchronous update on all linked flows for curve_name, flow in chart._flows.items(): + + if ( + not (do_rt_update or do_append) + and liv + # even if we're downsampled bigly + # draw the last datum in the final + # px column to give the user the mx/mn + # range of that set. + ): + # always update the last datum-element + # graphic for all flows + flow.draw_last(array_key=curve_name) + # TODO: should the "main" (aka source) flow be special? if curve_name == chart.data_key: continue diff --git a/piker/ui/_flows.py b/piker/ui/_flows.py index 7960d6499..01bbbece0 100644 --- a/piker/ui/_flows.py +++ b/piker/ui/_flows.py @@ -34,6 +34,7 @@ from numpy.lib import recfunctions as rfn import pyqtgraph as pg from PyQt5.QtGui import QPainterPath +from PyQt5.QtCore import QLineF from ..data._sharedmem import ( ShmArray, @@ -335,12 +336,6 @@ class Flow(msgspec.Struct): # , frozen=True): graphics: Curve _shm: ShmArray - draw_last: Optional[ - Callable[ - [np.ndarray, str], - tuple[np.ndarray] - ] - ] = None is_ohlc: bool = False render: bool = True # toggle for display loop @@ -594,7 +589,6 @@ def update_graphics( # TODO: append logic inside ``.render()`` isn't # correct yet for step curves.. remove this to see it. should_redraw = True - # draw_last = True slice_to_head = -2 # downsampling incremental state checking @@ -690,8 +684,47 @@ def update_graphics( graphics.update() profiler('.update()') + # track downsampled state + self._in_ds = r._in_ds + return graphics + def draw_last( + self, + array_key: Optional[str] = None, + + ) -> None: + + # shm read and slice to view + ( + xfirst, xlast, src_array, + ivl, ivr, in_view, + ) = self.read() + + g = self.graphics + array_key = array_key or self.name + x, y = g.draw_last_datum( + g.path, + src_array, + src_array, + False, # never reset path + array_key, + ) + + if self._in_ds: + # we only care about the last pixel's + # worth of data since that's all the screen + # can represent on the last column where + # the most recent datum is being drawn. + uppx = self._last_uppx + y = y[-uppx:] + ymn, ymx = y.min(), y.max() + # print(f'drawing uppx={uppx} mxmn line: {ymn}, {ymx}') + g._last_line = QLineF( + x[-2], ymn, + x[-1], ymx, + ) + def by_index_and_key( renderer: Renderer, From 44c242a7943cd818722710a9a9bc53ab56871510 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 5 Jun 2022 22:01:37 -0400 Subject: [PATCH 113/113] Fill in label with pairs from `status` value of backend init msg --- piker/data/feed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 94c2f81d6..1165fddca 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -47,7 +47,6 @@ import numpy as np from ..brokers import get_brokermod -from .._cacheables import maybe_open_context from ..calc import humanize from ..log import get_logger, get_console_log from .._daemon import (