feat(scripts): panel serve

- add pycharm .idea to .gitignore
- reorganize submodules structure
  - load data once centrally
  - distribute to gui instances per user session
  - add bondia.server
  - add bondia.data
- add bondia.util.exception
- disable buttons in template

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# PyCharm
+.idea

bondia/data.py

@@ -0,0 +1,72 @@
+from caput.config import Property, Reader
+from ch_pipeline.core import containers
+import glob
+import logging
+import numpy as np
+import os
+from pathlib import Path
+
+from .util.day import Day
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class DataLoader(Reader):
+    delay_spectrum = Property(proptype=Path)
+
+    def __init__(self):
+        self._index = {}
+
+    def _finalise_config(self):
+        """Do things after caput config reader is done."""
+        if self.delay_spectrum:
+            self.index_files(self.delay_spectrum)
+        else:
+            logger.debug("No path to delay spectrum data in config, skipping...")
+
+    @property
+    def index(self):
+        return self._index
+
+    def index_files(self, dirs):
+        """
+        (Re)index delay spectrum files.
+
+        Parameters
+        ----------
+        dirs : str or list(str)
+
+        Returns
+        -------
+        list(Day)
+            List of new lsd found.
+        """
+        if isinstance(dirs, os.PathLike):
+            dirs = [dirs]
+        files = []
+        for d in dirs:
+            files += sorted(glob.glob(os.path.join(d, "delayspectrum_lsd_*.h5")))
+        logger.debug("Found files: {}".format(files))
+
+        lsd = np.array(
+            [int(os.path.splitext(os.path.basename(ff))[0][-4:]) for ff in files]
+        )
+        new_lsd = []
+
+        for cc, filename in zip(lsd, files):
+            if cc not in self._index:
+                cc = Day.from_lsd(cc)
+                logger.info(f"Found new data for day {cc}.")
+                self._index[cc] = filename
+                new_lsd.append(cc)
+
+        return new_lsd
+
+    def load_file(self, day: Day):
+        """Load the delay spectrum of one day from a file."""
+        if isinstance(self._index[day], containers.DelaySpectrum):
+            return self._index[day]
+        logger.info(f"Loading day {day}.")
+        self._index[day] = containers.DelaySpectrum.from_file(self._index[day])
+        return self._index[day]

bondia/gui.py

@@ -1,47 +1,23 @@
-from bokeh.themes.theme import Theme
-from caput.config import Property, Reader
-import datetime
-import holoviews as hv
-from jinja2 import Environment, FileSystemLoader
 import panel as pn
-import pathlib
 
-from .plots.delayspectrum import DelaySpectrumPlot
+from .plot.delayspectrum import DelaySpectrumPlot
 
 
-class BondiaGui(Reader):
-    config_delayspectrum = Property(None, dict, "delay_spectrum")
-    template_name = Property("mwc", str, "html_template")
-    width_drawer_widgets = Property(220, int)
+class BondiaGui:
+    def __init__(self, template, width_drawer_widgets, data_loader):
+        self._width_drawer_widgets = width_drawer_widgets
+        self._template = template
+        self._plot = {}
+        self._toggle_plot = {}
+        self._data = data_loader
 
-    def __init__(self):
-        hv.extension("bokeh")
-        hv.renderer("bokeh").theme = Theme(json={})  # Reset Theme
-        pn.extension()
-
-        env = Environment(
-            loader=FileSystemLoader(pathlib.Path(__file__).parent / "templates")
-        )
-        jinja_template = env.get_template(f"{self.template_name}.html")
-
-        self.template = pn.Template(jinja_template)
-
-        self.template.add_variable("subtitle", "CHIME Daily Validation")
-        self.template.add_variable("app_title", "bon dia")
-
-        self.plot = {}
-        self.toggle_plot = {}
-
-    def populate_template(self):
-        if self.config_delayspectrum:
-            delay = DelaySpectrumPlot.from_config(self.config_delayspectrum)
-            self.plot[delay.id] = delay
+    def populate_template(self, template):
+        delay = DelaySpectrumPlot(self._data)
+        self._plot[delay.id] = delay
 
         # TODO: keep available days outside plot
         day_selector = pn.widgets.Select(
-            name="Select LSD",
-            options=list(delay.index.keys()),
-            width=self.width_drawer_widgets,
+            options=list(self._data.index.keys()), width=self._width_drawer_widgets,
         )
 
         # Set initial value
@@ -54,34 +30,39 @@ def populate_template(self):
         components = [("day_selector", day_selector)]
 
         # Fill in the plot selection toggle buttons
-        for p in self.plot.values():
-            self.toggle_plot[p.id] = pn.widgets.Toggle(
+        for p in self._plot.values():
+            self._toggle_plot[p.id] = pn.widgets.Toggle(
                 name=f"Deactivate {p.name}",
                 button_type="success",
                 value=True,
-                width=self.width_drawer_widgets,
+                width=self._width_drawer_widgets,
             )
-            self.toggle_plot[p.id].param.watch(self.update_widget, "value")
-            self.toggle_plot[p.id].param.trigger("value")
+            self._toggle_plot[p.id].param.watch(self.update_widget, "value")
+            self._toggle_plot[p.id].param.trigger("value")
 
-            components.append((f"toggle_{p.id}", self.toggle_plot[p.id]))
+            components.append((f"toggle_{p.id}", self._toggle_plot[p.id]))
             components.append((f"plot_{p.id}", p.panel_row))
 
         for l, c in components:
-            self.template.add_panel(l, c)
+            template.add_panel(l, c)
+        return template
+
+    def render(self):
+        template = pn.Template(self._template)
+
+        template.add_variable("subtitle", "CHIME Daily Validation")
+        template.add_variable("app_title", "bon dia")
 
-    def start_server(self):
-        self.template.show(port=8008)
+        return self.populate_template(template)
 
     def update_widget(self, event):
-        print(event)
         id = "delay_spectrum"
-        toggle = self.toggle_plot[id]
+        toggle = self._toggle_plot[id]
         if event.new:
-            self.plot[id].panel_row = True
+            self._plot[id].panel_row = True
             toggle.button_type = "success"
             toggle.name = "Deactivate Delay Spectrum"
         else:
-            self.plot[id].panel_row = False
+            self._plot[id].panel_row = False
             toggle.button_type = "danger"
             toggle.name = "Activate Delay Spectrum"

bondia/plot/delayspectrum.py

@@ -0,0 +1,176 @@
+import logging
+from matplotlib import cm as matplotlib_cm
+import panel
+import param
+import holoviews as hv
+from holoviews.operation.datashader import datashade
+from holoviews.plotting.util import process_cmap
+import numpy as np
+
+from .plot import BondiaPlot
+
+logger = logging.getLogger(__name__)
+
+
+class DelaySpectrumPlot(param.Parameterized, BondiaPlot):
+    """
+    Attributes
+    ----------
+    lsd : int
+        Local stellar day.
+    transpose
+        Transpose the plot if True. Default `False`.
+    log
+        True for logarithmic color map (z-values). Default `True`.
+    colormap_range
+        Range for the colormap (z-values).
+    serverside_rendering
+        True to use datashader. Automatically selects colormap for every zoom level, sends
+        pre-rendered images to client. Default `True`.
+    colormap_range
+        (optional, if using datashader) Select limits of color map values (z-values). Default
+        `None`.
+    """
+
+    # parameters
+    transpose = param.Boolean(default=False)
+    logarithmic_colorscale = param.Boolean(default=True)
+    helper_lines = param.Boolean(default=True)
+
+    # Default: turn on datashader and disable colormap range
+    serverside_rendering = param.Boolean(default=True)
+    colormap_range = param.Range(default=(0.1, 10000), constant=True)
+
+    # Hide lsd selector by setting precedence < 0
+    lsd = param.Selector(precedence=-1)
+
+    def panel(self):
+        return panel.Column(self.title, self.view)
+
+    def __init__(self, data, **params):
+        self.data = data
+        self.selections = None
+        BondiaPlot.__init__(self, "Delay Spectrum")
+        param.Parameterized.__init__(self, **params)
+
+    @param.depends("serverside_rendering", watch=True)
+    def update_serverside_rendering(self):
+        # Disable colormap range selection if using datashader (because it uses auto values)
+        self.param["colormap_range"].constant = self.serverside_rendering
+
+    @param.depends(
+        "lsd",
+        "transpose",
+        "logarithmic_colorscale",
+        "serverside_rendering",
+        "colormap_range",
+        "helper_lines",
+    )
+    def view(self):
+        spectrum = self.data.load_file(self.lsd)
+        x, y = spectrum.index_map["baseline"].T
+
+        # Index map for delay (x-axis)
+        index_map_delay_nsec = spectrum.index_map["delay"] * 1e3
+
+        ux, uix = np.unique(np.round(x).astype(np.int), return_inverse=True)
+
+        nplot = ux.size
+
+        # Fill a row with plots (one per pair of cylinders)
+        all_img = panel.Row()
+        mplot = {}
+        ylim = None
+        for pp, pux in reversed(list(enumerate(ux))):
+            # Get the baselines for this cylinder separation
+            this_cyl_sep = np.flatnonzero(uix == pp)
+
+            # Sort the indices by the N-S baseline distance
+            this_cyl_sep = this_cyl_sep[np.argsort(y[this_cyl_sep])]
+
+            # Determine span of data (y-axis)
+            range_y = np.percentile(y[this_cyl_sep], [0, 100])
+            range_x = np.percentile(index_map_delay_nsec, [0, 100])
+
+            # Discard baselines that are set to zero
+            this_cyl_sep = this_cyl_sep[
+                np.any(spectrum.spectrum[this_cyl_sep, :] > 0.0, axis=-1)
+            ]
+
+            # Index map for baseline (y-axis)
+            baseline_index = y[this_cyl_sep]
+
+            # Plot for south-west baseline == 0 is done last. Keep the same y-axis range for it.
+            if pux != 0 or ylim is None:
+                ylim_max = (range_y[0], range_y[-1])
+            xlim = (range_x[0], range_x[-1])
+            ylim = ylim_max
+
+            # Make image
+            if self.transpose:
+                mplot[pp] = spectrum.spectrum[this_cyl_sep, :].T
+                index_x = baseline_index
+                index_y = index_map_delay_nsec
+                xlim, ylim = ylim, xlim
+
+            else:
+                mplot[pp] = spectrum.spectrum[this_cyl_sep, :]
+                index_x = index_map_delay_nsec
+                index_y = baseline_index
+
+            # holoviews checks for regular sampling before plotting an Image.
+            # The CHIME baselines are not regularly sampled enough to pass through the default rtol
+            # (1e-6), but we anyways want to plot the delay spectrum in an Image, not a QuadMesh.
+            img = hv.Image(
+                (index_x, index_y, mplot[pp]),
+                datatype=["image", "grid"],
+                kdims=["τ [nsec]", "y [m]"],
+                rtol=2,
+            ).opts(
+                clim=self.colormap_range,
+                logz=self.logarithmic_colorscale,
+                cmap=process_cmap("inferno", provider="matplotlib"),
+                colorbar=(pp == nplot - 1),
+                title=f"x = {pux} m",
+                xlim=xlim,
+                ylim=ylim,
+            )
+
+            if not self.serverside_rendering:
+                if self.helper_lines:
+                    img = (img * hv.VLine(0) * hv.HLine(0)).opts(
+                        hv.opts.VLine(color="white", line_width=3, line_dash="dotted"),
+                        hv.opts.HLine(color="white", line_width=3, line_dash="dotted"),
+                    )
+                all_img.insert(0, img)
+            else:
+                # set colormap
+                cmap_inferno = matplotlib_cm.__dict__["inferno"]
+                cmap_inferno.set_under("black")
+                cmap_inferno.set_bad("lightgray")
+
+                # Set z-axis normalization (other possible values are 'eq_hist', 'cbrt').
+                if self.logarithmic_colorscale:
+                    normalization = "log"
+                else:
+                    normalization = "linear"
+
+                # datashade
+                img = datashade(
+                    img,
+                    cmap=cmap_inferno,
+                    precompute=True,
+                    x_range=xlim,
+                    y_range=ylim,
+                    normalization=normalization,
+                )
+
+                if self.helper_lines:
+                    img = (img * hv.VLine(0) * hv.HLine(0)).opts(
+                        hv.opts.VLine(color="white", line_width=3, line_dash="dotted"),
+                        hv.opts.HLine(color="white", line_width=3, line_dash="dotted"),
+                    )
+
+                all_img.insert(0, img)
+
+        return all_img

bondia/plots/plot.py → bondia/plot/plot.py

@@ -2,9 +2,9 @@
 
 
 class BondiaPlot:
-    def __init__(self, title: str, activated: bool = True):
-        self._name = title
-        self._id = title.lower().replace(" ", "_")
+    def __init__(self, name: str, activated: bool = True):
+        self._name = name
+        self._id = name.lower().replace(" ", "_")
         self._panel_row_active = activated
         self._panel_row = None
 
@@ -16,6 +16,10 @@ def id(self):
     def name(self):
         return self._name
 
+    @property
+    def title(self):
+        return f"## {self._name}"
+
     @property
     def panel_row(self):
         if self._panel_row is None: