diff --git a/.gitignore b/.gitignore index aa965d7d..3078791b 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,12 @@ cython_debug/ # sphinx-docs sphinx/_build sphinx/_autosummary + +#testing stuff +logs/ +selectionbox_layout_data/ +examples/dash_apps/**/*coarse_fine* +examples/dash_apps/2* +examples/dash_apps/00* +tests/log_processing*.ipynb +figure data/ diff --git a/examples/dash_apps/01_minimal_global.py b/examples/dash_apps/01_minimal_global.py index 0a7ab08f..a39fc9cb 100644 --- a/examples/dash_apps/01_minimal_global.py +++ b/examples/dash_apps/01_minimal_global.py @@ -16,18 +16,21 @@ import numpy as np import plotly.graph_objects as go from dash import Dash, Input, Output, callback_context, dcc, html, no_update + +# from graph_reporter import GraphReporter from trace_updater import TraceUpdater from plotly_resampler import FigureResampler # Data that will be used for the plotly-resampler figures -x = np.arange(2_000_000) +n = 500_000 +x = np.arange(n) noisy_sin = (3 + np.sin(x / 200) + np.random.randn(len(x)) / 10) * x / 1_000 - +flat = np.ones(n) # --------------------------------------Globals --------------------------------------- app = Dash(__name__) -fig: FigureResampler = FigureResampler() +fig: FigureResampler = FigureResampler(verbose=False) # NOTE: in this example, this reference to a FigureResampler is essential to preserve # throughout the whole dash app! If your dash app wants to create a new go.Figure(), # you should not construct a new FigureResampler object, but replace the figure of this @@ -39,8 +42,11 @@ html.Button("plot chart", id="plot-button", n_clicks=0), html.Hr(), # The graph and it's needed components to update efficiently + dcc.Store(id="visible-indices", data={"visible": [], "invisible": []}), dcc.Graph(id="graph-id"), TraceUpdater(id="trace-updater", gdID="graph-id"), + # GraphReporter(id="graph-reporter", gId="graph-id"), + # html.Div(id='print') ] ) @@ -49,6 +55,7 @@ # The callback used to construct and store the graph's data on the serverside @app.callback( Output("graph-id", "figure"), + # Output("visible-indices", "data"), Input("plot-button", "n_clicks"), prevent_initial_call=True, ) @@ -62,14 +69,22 @@ def plot_graph(n_clicks): fig.replace(go.Figure()) fig.add_trace(go.Scattergl(name="log"), hf_x=x, hf_y=noisy_sin * 0.9999995**x) fig.add_trace(go.Scattergl(name="exp"), hf_x=x, hf_y=noisy_sin * 1.000002**x) + fig.add_trace(go.Scattergl(name="const"), hf_x=x, hf_y=flat) + fig.add_trace(go.Scattergl(name="poly"), hf_x=x, hf_y=noisy_sin * 1.000002**2) + + # fig._print_verbose = True + fig.update_layout(showlegend=True) return fig else: - return no_update + return no_update, no_update # Register the graph update callbacks to the layout fig.register_update_graph_callback( - app=app, graph_id="graph-id", trace_updater_id="trace-updater" + app=app, + graph_id="graph-id", + trace_updater_id="trace-updater", + visibility_store_id="visible-indices", ) # --------------------------------- Running the app --------------------------------- diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json new file mode 100644 index 00000000..b4295f14 --- /dev/null +++ b/node_modules/.package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "plotly-resampler", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + } + } +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..6d19a7b1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,27 @@ +{ + "name": "plotly-resampler", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "plotly-resampler", + "version": "1.0.0", + "dependencies": { + "lodash": "^4.17.21" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + } + }, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..5573e938 --- /dev/null +++ b/package.json @@ -0,0 +1,7 @@ +{ + "name": "plotly-resampler", + "version": "1.0.0", + "dependencies": { + "lodash": "^4.17.21" + } +} diff --git a/plotly_resampler/figure_resampler/figure_resampler.py b/plotly_resampler/figure_resampler/figure_resampler.py index b9beff5f..acb3f868 100644 --- a/plotly_resampler/figure_resampler/figure_resampler.py +++ b/plotly_resampler/figure_resampler/figure_resampler.py @@ -6,6 +6,7 @@ """ + from __future__ import annotations __author__ = "Jonas Van Der Donckt, Jeroen Van Der Donckt, Emiel Deprost" @@ -15,6 +16,8 @@ import dash import plotly.graph_objects as go + +# from graph_reporter import GraphReporter from plotly.basedatatypes import BaseFigure from trace_updater import TraceUpdater @@ -200,6 +203,7 @@ def show_dash( self, mode=None, config: dict | None = None, + testing: bool | None = False, graph_properties: dict | None = None, **kwargs, ): @@ -298,15 +302,24 @@ def show_dash( app = dash.Dash("local_app") app.layout = dash.html.Div( [ + dash.dcc.Store( + id="visible-indices", data={"visible": [], "invisible": []} + ), dash.dcc.Graph( id="resample-figure", figure=self, config=config, **graph_properties ), TraceUpdater( - id="trace-updater", gdID="resample-figure", sequentialUpdate=False + id="trace-updater", + gdID="resample-figure", + sequentialUpdate=False, + verbose=testing, ), + # GraphReporter(id="graph-reporter", gId="resample-figure"), ] ) - self.register_update_graph_callback(app, "resample-figure", "trace-updater") + self.register_update_graph_callback( + app, "resample-figure", "trace-updater", "visible-indices" + ) height_param = "height" if self._is_persistent_inline else "jupyter_height" @@ -365,8 +378,14 @@ def stop_server(self, warn: bool = True): + "\t- the dash-server wasn't started with 'show_dash'" ) + # TODO: check if i should put the clientside callback to fill the store here or in a different function + # for now, here def register_update_graph_callback( - self, app: dash.Dash, graph_id: str, trace_updater_id: str + self, + app: dash.Dash, + graph_id: str, + trace_updater_id: str, + visibility_store_id: str, ): """Register the [`construct_update_data`][figure_resampler.figure_resampler_interface.AbstractFigureAggregator.construct_update_data] method as callback function to the passed dash-app. @@ -382,14 +401,83 @@ def register_update_graph_callback( The id of the ``TraceUpdater`` component. This component is leveraged by ``FigureResampler`` to efficiently POST the to-be-updated data to the front-end. + visibility_store_id + The id of the ``dcc.Store`` component which holds the indices of the visible + traces in the client. Leveraged to efficiently perform the asynchronous update of + the visible and invisible traces of the ``Graph``. """ + # Callback triggers when a stylistic change is made to the graph + # this includes hiding traces or making them visible again, which is the + # desired use-case + app.clientside_callback( + """ + function(restyleData, gdID) { + // HELPER FUNCTIONS + + function getGraphDiv(gdID){ + // see this link for more information https://stackoverflow.com/a/34002028 + let graphDiv = document?.querySelectorAll('div[id*="' + gdID + '"][class*="dash-graph"]'); + if (graphDiv.length > 1) { + throw new SyntaxError("UpdateStore: multiple graphs with ID=" + gdID + " found; n=" + graphDiv.length + " (either multiple graphs with same ID's or current ID is a str-subset of other graph IDs)"); + } else if (graphDiv.length < 1) { + throw new SyntaxError("UpdateStore: no graphs with ID=" + gdID + " found"); + } + graphDiv = graphDiv?.[0]?.getElementsByClassName('js-plotly-plot')?.[0]; + const isDOMElement = el => el instanceof HTMLElement + if (!isDOMElement) { + throw new Error(`Invalid gdID '${gdID}'`); + } + return graphDiv; + } + + //MAIN CALLBACK + let storeData = {'visible':[], 'invisible':[]}; + if (restyleData) { + let graphDiv = getGraphDiv(gdID); + + //console.log("restyleData:"); + //console.log(restyleData); + //console.log("\tgraph data -> visibility of traces: "); + + let visible_traces = []; + let invisible_traces = []; + graphDiv.data.forEach((trace, index) => { + //console.log('\tvisible: ' + trace.visible); + if (trace.visible == true || trace.visible == undefined) { + visible_traces.push(index); + } else { + invisible_traces.push(index); + } + }); + storeData = {'visible':visible_traces, 'invisible':invisible_traces}; + } + //console.log(storeData); + return storeData; + } + """, + dash.dependencies.Output(visibility_store_id, "data"), + dash.dependencies.Input(graph_id, "restyleData"), + dash.dependencies.State(graph_id, "id"), + ) + app.callback( dash.dependencies.Output(trace_updater_id, "updateData"), dash.dependencies.Input(graph_id, "relayoutData"), + # dash.dependencies.State(graph_id, "restyleData"), + dash.dependencies.State(visibility_store_id, "data"), prevent_initial_call=True, )(self.construct_update_data) + app.callback( + dash.dependencies.Output(trace_updater_id, "invisibleUpdateData"), + # dash.dependencies.Input(trace_updater_id, "updateData"), + dash.dependencies.Input(trace_updater_id, "visibleUpdate"), + dash.dependencies.State(graph_id, "relayoutData"), + dash.dependencies.State(visibility_store_id, "data"), + prevent_initial_call=True, + )(self.construct_invisible_update_data) + def _get_pr_props_keys(self) -> List[str]: # Add the additional plotly-resampler properties of this class return super()._get_pr_props_keys() + ["_show_dash_kwargs"] diff --git a/plotly_resampler/figure_resampler/figure_resampler_interface.py b/plotly_resampler/figure_resampler/figure_resampler_interface.py index 4c9b7d1f..65626d9c 100644 --- a/plotly_resampler/figure_resampler/figure_resampler_interface.py +++ b/plotly_resampler/figure_resampler/figure_resampler_interface.py @@ -57,6 +57,7 @@ def __init__( show_mean_aggregation_size: bool = True, convert_traces_kwargs: dict | None = None, verbose: bool = False, + # TODO: add c_width parameter ): """Instantiate a resampling data mirror. @@ -271,7 +272,7 @@ def _check_update_trace_data( start: Optional[Union[str, float]] = None, end: Optional[Union[str, float]] = None, ) -> Optional[Union[dict, BaseTraceType]]: - """Check and update the passed ``trace`` its data properties based on the + """Check and update the passed ddfkj``trace`` its data properties based on the slice range. Note @@ -426,6 +427,7 @@ def _check_update_figure_dict( stop: Optional[Union[float, str]] = None, layout_xaxis_filter: Optional[str] = None, updated_trace_indices: Optional[List[int]] = None, + indices_to_use: Optional[List[int]] = None, ) -> List[int]: """Check and update the traces within the figure dict. @@ -463,17 +465,26 @@ def _check_update_figure_dict( if updated_trace_indices is None: updated_trace_indices = [] + if indices_to_use is None: + indices_to_use = [] + if layout_xaxis_filter is not None: layout_trace_mapping = self._layout_xaxis_to_trace_xaxis_mapping() # Retrieve the trace xaxis values that are affected by the relayout event trace_xaxis_filter: List[str] = layout_trace_mapping[layout_xaxis_filter] for idx, trace in enumerate(figure["data"]): - # We skip when (i) the trace-idx already has been updated or (ii) when + # We skip when the trace-idx already has been updated or when it's not due for an update. + # We skip when (i) the trace-idx already has been updated, (ii) when # there is a layout_xaxis_filter and the trace xaxis is not in the filter - if idx in updated_trace_indices or ( - layout_xaxis_filter is not None - and trace.get("xaxis", "x") not in trace_xaxis_filter + # or (iii) when its not part of the subset of traces (visible / invisible) to update now + if ( + idx in updated_trace_indices + or idx not in indices_to_use + or ( + layout_xaxis_filter is not None + and trace.get("xaxis", "x") not in trace_xaxis_filter + ) ): continue @@ -490,7 +501,10 @@ def _get_figure_class(constr: type) -> type: !!! note This method will always return a plotly constructor, even when the given - `constr` is decorated (after executing the ``register_plotly_resampler`` + `constr` is decorated (after executing the `` + + + _plotly_resampler`` function). Parameters @@ -1290,8 +1304,7 @@ def _parse_relayout(self, relayout_dict: dict) -> dict: return extra_layout_updates def construct_update_data( - self, - relayout_data: dict, + self, relayout_data: dict, trace_visibility: dict ) -> Union[List[dict], dash.no_update]: """Construct the to-be-updated front-end data, based on the layout change. @@ -1308,6 +1321,12 @@ def construct_update_data( A dict containing the ``relayout``-data (a.k.a. changed layout data) of the corresponding front-end graph. + figure: dict + A dict containing the ``figure``-data (a.k.a. all the data needed to plot traces and style them accordingly) of + the corresponding front-end graph. Used to determine the current visible state of each trace + NOTE: there should be a better way to pass ONLY the visible state of the traces to the back-end + wrap dcc.Graph? => could extract the visible data from the figure before passing it to the callback? + Returns ------- List[dict]: @@ -1319,6 +1338,18 @@ def construct_update_data( in each dict. """ + if ( + len(trace_visibility["visible"]) == 0 + and len(trace_visibility["invisible"]) == 0 + ): + visible_trace_idx = [i for i, trace in enumerate(self._data)] + else: + visible_trace_idx = trace_visibility["visible"] + + # import json + # import datetime + # with open(f'figure_{datetime.datetime.now().strftime("%H_%M")}.json', 'w') as f: + # json.dump({"data": figure['data']}, f) current_graph = self._get_current_graph() updated_trace_indices, cl_k = [], [] if relayout_data: @@ -1337,11 +1368,12 @@ def construct_update_data( assert xaxis == t_stop_key.split(".")[0] # -> we want to copy the layout on the back-end updated_trace_indices = self._check_update_figure_dict( - current_graph, + figure=current_graph, start=relayout_data[t_start_key], stop=relayout_data[t_stop_key], layout_xaxis_filter=xaxis, updated_trace_indices=updated_trace_indices, + indices_to_use=visible_trace_idx, ) # 2. The user clicked on either autorange | reset axes @@ -1358,6 +1390,7 @@ def construct_update_data( current_graph, layout_xaxis_filter=xaxis, updated_trace_indices=updated_trace_indices, + indices_to_use=visible_trace_idx, ) # 2.1. Autorange -> do nothing, the autorange will be applied on the # current front-end view @@ -1393,6 +1426,93 @@ def construct_update_data( layout_traces_list.append(trace_reduced) return layout_traces_list + def construct_invisible_update_data( + self, visible_update: int, relayout_data, trace_visibility: dict + ) -> Union[List[dict], dash.no_update]: + invisible_trace_idx = trace_visibility["invisible"] + + current_graph = self._get_current_graph() + updated_trace_indices, cl_k = [], [] + if relayout_data: + self._print("-" * 100 + "\n", "changed layout", relayout_data) + + cl_k = relayout_data.keys() + + # ------------------ HF DATA aggregation --------------------- + # 1. Base case - there is a x-range specified in the front-end + start_matches = self._re_matches(re.compile(r"xaxis\d*.range\[0]"), cl_k) + stop_matches = self._re_matches(re.compile(r"xaxis\d*.range\[1]"), cl_k) + if len(start_matches) and len(stop_matches): + for t_start_key, t_stop_key in zip(start_matches, stop_matches): + # Check if the xaxis part of xaxis.[0-1] matches + xaxis = t_start_key.split(".")[0] + assert xaxis == t_stop_key.split(".")[0] + # -> we want to copy the layout on the back-end + updated_trace_indices = self._check_update_figure_dict( + figure=current_graph, + start=relayout_data[t_start_key], + stop=relayout_data[t_stop_key], + layout_xaxis_filter=xaxis, + updated_trace_indices=updated_trace_indices, + indices_to_use=invisible_trace_idx, + ) + + # 2. The user clicked on either autorange | reset axes + autorange_matches = self._re_matches( + re.compile(r"xaxis\d*.autorange"), cl_k + ) + spike_matches = self._re_matches(re.compile(r"xaxis\d*.showspikes"), cl_k) + # 2.1 Reset-axes -> autorange & reset to the global data view + if len(autorange_matches) and len(spike_matches): + for autorange_key in autorange_matches: + if relayout_data[autorange_key]: + xaxis = autorange_key.split(".")[0] + updated_trace_indices = self._check_update_figure_dict( + current_graph, + layout_xaxis_filter=xaxis, + updated_trace_indices=updated_trace_indices, + indices_to_use=invisible_trace_idx, + ) + # 2.1. Autorange -> do nothing, the autorange will be applied on the + # current front-end view + elif len(autorange_matches) and not len(spike_matches): + # PreventUpdate returns a 204 status code response on the + # relayout post request + return dash.no_update + + # If we do not have any traces to be updated, we will return an empty + # request response + if len(updated_trace_indices) == 0: + # PreventUpdate returns a 204 status-code response on the relayout post + # request + return dash.no_update + + # -------------------- construct callback data -------------------------- + layout_traces_list: List[dict] = [] # the data + + # 1. Create a new dict with additional layout updates for the front-end + extra_layout_updates = {} + + # 1.1. Set autorange to False for each layout item with a specified x-range + xy_matches = self._re_matches(re.compile(r"[xy]axis\d*.range\[\d+]"), cl_k) + for range_change_axis in xy_matches: + axis = range_change_axis.split(".")[0] + extra_layout_updates[f"{axis}.autorange"] = False + layout_traces_list.append(extra_layout_updates) + + # 2. Create the additional trace data for the frond-end + relevant_keys = ["x", "y", "text", "hovertext", "name"] # TODO - marker color + # Note that only updated trace-data will be sent to the client + for idx in updated_trace_indices: + trace = current_graph["data"][idx] + trace_reduced = {k: trace[k] for k in relevant_keys if k in trace} + + # Store the index into the corresponding to-be-sent trace-data so + # the client front-end can know which trace needs to be updated + trace_reduced.update({"index": idx}) + layout_traces_list.append(trace_reduced) + return layout_traces_list + @staticmethod def _parse_dtype_orjson(series: np.ndarray) -> np.ndarray: """Verify the orjson compatibility of the series and convert it if needed.""" @@ -1412,6 +1532,7 @@ def _re_matches(regex: re.Pattern, strings: Iterable[str]) -> List[str]: m = regex.match(item) if m is not None: matches.append(m.string) + # print(f'sorted(matches): {sorted(matches)}') return sorted(matches) @staticmethod diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/conftest.py b/tests/conftest.py index 0cb83543..30a770a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ # hyperparameters _nb_samples = 10_000 data_dir = "examples/data/" -headless = True +headless = False TESTING_LOCAL = False # SET THIS TO TRUE IF YOU ARE TESTING LOCALLY @@ -58,6 +58,7 @@ def driver(): options = Options() d = DesiredCapabilities.CHROME d["goog:loggingPrefs"] = {"browser": "ALL"} + d["acceptSslCerts"] = True if not TESTING_LOCAL: if headless: options.add_argument("--headless") diff --git a/tests/fr_selenium.py b/tests/fr_selenium.py index 445713f7..bb9076cb 100644 --- a/tests/fr_selenium.py +++ b/tests/fr_selenium.py @@ -200,7 +200,9 @@ def get_requests(self, delete: bool = True): return requests - def drag_and_zoom(self, div_classname, x0=0.25, x1=0.5, y0=0.25, y1=0.5): + def drag_and_zoom( + self, div_classname, x0=0.25, x1=0.5, y0=0.25, y1=0.5, testing=False + ): """ Drags and zooms the div with the given classname. @@ -232,13 +234,20 @@ def drag_and_zoom(self, div_classname, x0=0.25, x1=0.5, y0=0.25, y1=0.5): actions = ActionChains(self.driver) actions.move_to_element_with_offset(subplot, xoffset=w * x0, yoffset=h * y0) actions.click_and_hold() - actions.pause(0.2) + actions.pause(0.1) actions.move_by_offset(xoffset=w * (x1 - x0), yoffset=h * (y1 - y0)) - actions.pause(0.2) - actions.release() - actions.pause(0.2) + actions.pause(0.1) actions.perform() + action = ActionChains(self.driver) + action.release() + if testing: + # self.driver.execute_script("console.log('time update visible');") + self.driver.execute_script( + "console.time('time (visible)');console.time('time (full)');" + ) + action.perform() + def _get_modebar_btns(self): if not self.on_page: self.go_to_page() @@ -255,11 +264,21 @@ def autoscale(self): ActionChains(self.driver).move_to_element(btn).click().perform() return - def reset_axes(self): + def reset_axes(self, testing=False): for btn in self._get_modebar_btns(): data_title = btn.get_attribute("data-title") if data_title == "Reset axes": - ActionChains(self.driver).move_to_element(btn).click().perform() + ActionChains(self.driver).move_to_element(btn).perform() + + # NOTE: execucte the click right after the log + actions = ActionChains(self.driver) + actions.click() + if testing: + # self.driver.execute_script("console.log('time update visible');") + self.driver.execute_script( + "console.time('time (visible)');console.time('time (full)');" + ) + actions.perform() return def click_legend_item(self, legend_name): @@ -278,6 +297,23 @@ def click_legend_item(self, legend_name): ) return + def hide_legend_restyle(self, item_numbers): + + # for the moment this only works with 1 graph present? + graph = self.driver.find_elements(by=By.CLASS_NAME, value="js-plotly-plot") + # TODO: find way to scroll down to an element (trace in legend) within an element (legend) + self.driver.execute_script( + "Plotly.restyle(arguments[0], {'visible': ['legendonly']},arguments[1])", + graph[0], + item_numbers, + ) + + def start_timer(self, type): + if type == "zoom": + self.driver.execute_script("console.log('zoom in')") + else: + self.driver.execute_script("console.log('reset')") + # ------------------------------ DATA MODEL METHODS ------------------------------ def __del__(self): self.driver.close() diff --git a/tests/log_processing.ipynb b/tests/log_processing.ipynb new file mode 100644 index 00000000..f71ca0f7 --- /dev/null +++ b/tests/log_processing.ipynb @@ -0,0 +1,864 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import re\n", + "import os\n", + "\n", + "\n", + "# only needed if trailing object error\n", + "\n", + "# for filename in os.listdir('../logs/linux_jonas/vanilla_plotly_logs/'):\n", + "# f = os.path.join('../logs/linux_jonas/vanilla_plotly_logs/', filename)\n", + "# if os.path.isfile(f) & f.endswith(\".json\"):\n", + "# with open(f, 'r') as r:\n", + "# content = r.read()\n", + "# content = re.sub(r'}\\s*{', '},{', content)\n", + "# json_data = json.loads('[' + content + ']')\n", + "# with open(f, 'w') as w:\n", + "# json.dump(json_data, w)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 100.0%\n", + "1 time (visible): 1805.199951171875 ms\n", + "2 render time (visible): 827.38818359375 ms\n", + "3 time (invisible): 1.220947265625 ms\n", + "4 time (full): 1808.047119140625 ms\n", + " ... \n", + "65995 time (visible): 261.415771484375 ms\n", + "65996 render time (visible): 81.3662109375 ms\n", + "65997 time (invisible): 0.520751953125 ms\n", + "65998 time (full): 264.547607421875 ms\n", + "65999 render time (invisible): None\n", + "Name: message, Length: 66000, dtype: object" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "# df = pd.read_json('logs/n100000_s100_t10_everynth.json', orient ='index')\n", + "df = pd.DataFrame()\n", + "\n", + "\n", + "for filename in os.listdir('../logs/final/vanilla_pr_logs/'):\n", + " f = os.path.join('../logs/final/vanilla_pr_logs/', filename)\n", + " if os.path.isfile(f) & (filename.split('.')[1] == 'json'):\n", + " dft = pd.read_json(f)\n", + " dft['datapoints'] = filename.split('_')[0][1:]\n", + " dft['samples'] = filename.split('_')[1][1:]\n", + " dft['traces'] = filename.split('_')[2][1:]\n", + " dft['aggregator'] = filename.split('_')[3]\n", + " dft['iteration'] = filename.split(\"_\")[4].split('.')[0][-1:]\n", + " df = pd.concat([df, dft], ignore_index=True)\n", + " \n", + "\n", + "\n", + "df['message']= df['message'].str.split('\\\"').str[-2]\n", + "df.drop(df[df['level'].eq('WARNING')].index, inplace=True)\n", + "df = df.drop('source', axis=1)\n", + "df.reset_index()\n", + "df['message']\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
levelmessagetimestampdatapointssamplestracesaggregatoriteration
0INFO100.0%2023-08-11 17:04:16.2051000000010000100everynth0
1DEBUGtime (visible): 1942.76708984375 ms2023-08-11 17:04:19.2341000000010000100everynth0
2DEBUGrender time (visible): 942.869140625 ms2023-08-11 17:04:19.2361000000010000100everynth0
3DEBUGtime (invisible): 1.306884765625 ms2023-08-11 17:04:19.2371000000010000100everynth0
4DEBUGtime (full): 1945.81103515625 ms2023-08-11 17:04:19.2371000000010000100everynth0
...........................
65997DEBUGtime (visible): 168.304931640625 ms2023-08-12 00:45:01.208100000500050everynth9
65998DEBUGrender time (visible): 67.005126953125 ms2023-08-12 00:45:01.208100000500050everynth9
65999DEBUGtime (invisible): 220.11083984375 ms2023-08-12 00:45:01.428100000500050everynth9
66000DEBUGrender time (invisible): 72.118896484375 ms2023-08-12 00:45:01.429100000500050everynth9
66001DEBUGtime (full): 390.1201171875 ms2023-08-12 00:45:01.429100000500050everynth9
\n", + "

66000 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " level message \\\n", + "0 INFO 100.0% \n", + "1 DEBUG time (visible): 1942.76708984375 ms \n", + "2 DEBUG render time (visible): 942.869140625 ms \n", + "3 DEBUG time (invisible): 1.306884765625 ms \n", + "4 DEBUG time (full): 1945.81103515625 ms \n", + "... ... ... \n", + "65997 DEBUG time (visible): 168.304931640625 ms \n", + "65998 DEBUG render time (visible): 67.005126953125 ms \n", + "65999 DEBUG time (invisible): 220.11083984375 ms \n", + "66000 DEBUG render time (invisible): 72.118896484375 ms \n", + "66001 DEBUG time (full): 390.1201171875 ms \n", + "\n", + " timestamp datapoints samples traces aggregator iteration \n", + "0 2023-08-11 17:04:16.205 10000000 10000 100 everynth 0 \n", + "1 2023-08-11 17:04:19.234 10000000 10000 100 everynth 0 \n", + "2 2023-08-11 17:04:19.236 10000000 10000 100 everynth 0 \n", + "3 2023-08-11 17:04:19.237 10000000 10000 100 everynth 0 \n", + "4 2023-08-11 17:04:19.237 10000000 10000 100 everynth 0 \n", + "... ... ... ... ... ... ... \n", + "65997 2023-08-12 00:45:01.208 100000 5000 50 everynth 9 \n", + "65998 2023-08-12 00:45:01.208 100000 5000 50 everynth 9 \n", + "65999 2023-08-12 00:45:01.428 100000 5000 50 everynth 9 \n", + "66000 2023-08-12 00:45:01.429 100000 5000 50 everynth 9 \n", + "66001 2023-08-12 00:45:01.429 100000 5000 50 everynth 9 \n", + "\n", + "[66000 rows x 8 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_prio = pd.DataFrame()\n", + "\n", + "\n", + "for filename in os.listdir('../logs/final/visual_gain_logs/'):\n", + " f = os.path.join('../logs/final/visual_gain_logs/', filename)\n", + " if os.path.isfile(f) & f.endswith(\".json\"):\n", + " dft = pd.read_json(f)\n", + " dft['datapoints'] = filename.split('_')[0][1:]\n", + " dft['samples'] = filename.split('_')[1][1:]\n", + " dft['traces'] = filename.split('_')[2][1:]\n", + " dft['aggregator'] = filename.split('_')[3]\n", + " dft['iteration'] = filename.split(\"_\")[4].split('.')[0][-1:]\n", + " df_prio = pd.concat([df_prio, dft], ignore_index=True)\n", + " \n", + "df_prio['message']= df_prio['message'].str.split('\\\"').str[-2]\n", + "df_prio.drop(df_prio[df_prio['level'].eq('SEVERE')].index, inplace=True)\n", + "df_prio.drop('source', axis=1, inplace=True)\n", + "df_prio.reset_index()\n", + "df_prio" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
messagepercentagetimestamp
0time (visible): 1805.199951171875 ms100.0%2023-08-07 17:06:47.462
1render time (visible): 827.38818359375 ms100.0%2023-08-07 17:06:47.463
2time (invisible): 1.220947265625 ms100.0%2023-08-07 17:06:47.464
3time (full): 1808.047119140625 ms100.0%2023-08-07 17:06:47.464
4render time (invisible): None100.0%2023-08-07 17:06:47.464
............
59995time (visible): 261.415771484375 ms10.0%2023-08-08 00:53:52.332
59996render time (visible): 81.3662109375 ms10.0%2023-08-08 00:53:52.332
59997time (invisible): 0.520751953125 ms10.0%2023-08-08 00:53:52.332
59998time (full): 264.547607421875 ms10.0%2023-08-08 00:53:52.333
59999render time (invisible): None10.0%2023-08-08 00:53:52.333
\n", + "

60000 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " message percentage \\\n", + "0 time (visible): 1805.199951171875 ms 100.0% \n", + "1 render time (visible): 827.38818359375 ms 100.0% \n", + "2 time (invisible): 1.220947265625 ms 100.0% \n", + "3 time (full): 1808.047119140625 ms 100.0% \n", + "4 render time (invisible): None 100.0% \n", + "... ... ... \n", + "59995 time (visible): 261.415771484375 ms 10.0% \n", + "59996 render time (visible): 81.3662109375 ms 10.0% \n", + "59997 time (invisible): 0.520751953125 ms 10.0% \n", + "59998 time (full): 264.547607421875 ms 10.0% \n", + "59999 render time (invisible): None 10.0% \n", + "\n", + " timestamp \n", + "0 2023-08-07 17:06:47.462 \n", + "1 2023-08-07 17:06:47.463 \n", + "2 2023-08-07 17:06:47.464 \n", + "3 2023-08-07 17:06:47.464 \n", + "4 2023-08-07 17:06:47.464 \n", + "... ... \n", + "59995 2023-08-08 00:53:52.332 \n", + "59996 2023-08-08 00:53:52.332 \n", + "59997 2023-08-08 00:53:52.332 \n", + "59998 2023-08-08 00:53:52.333 \n", + "59999 2023-08-08 00:53:52.333 \n", + "\n", + "[60000 rows x 3 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mask = df['message'].str.contains('%')\n", + "percentage = df[mask]\n", + "percentage\n", + "df.loc[mask, 'percentage'] = percentage['message']\n", + "df['percentage'].fillna(method='ffill', inplace=True)\n", + "df.drop(percentage.index, inplace=True)\n", + "df = df.reset_index()\n", + "df[['message', 'percentage','timestamp']]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
messagepercentagetimestamp
0time (visible): 1942.76708984375 ms100.0%2023-08-11 17:04:19.234
1render time (visible): 942.869140625 ms100.0%2023-08-11 17:04:19.236
2time (invisible): 1.306884765625 ms100.0%2023-08-11 17:04:19.237
3time (full): 1945.81103515625 ms100.0%2023-08-11 17:04:19.237
4render time (invisible): None100.0%2023-08-11 17:04:19.240
............
59995time (visible): 168.304931640625 ms10.0%2023-08-12 00:45:01.208
59996render time (visible): 67.005126953125 ms10.0%2023-08-12 00:45:01.208
59997time (invisible): 220.11083984375 ms10.0%2023-08-12 00:45:01.428
59998render time (invisible): 72.118896484375 ms10.0%2023-08-12 00:45:01.429
59999time (full): 390.1201171875 ms10.0%2023-08-12 00:45:01.429
\n", + "

60000 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " message percentage \\\n", + "0 time (visible): 1942.76708984375 ms 100.0% \n", + "1 render time (visible): 942.869140625 ms 100.0% \n", + "2 time (invisible): 1.306884765625 ms 100.0% \n", + "3 time (full): 1945.81103515625 ms 100.0% \n", + "4 render time (invisible): None 100.0% \n", + "... ... ... \n", + "59995 time (visible): 168.304931640625 ms 10.0% \n", + "59996 render time (visible): 67.005126953125 ms 10.0% \n", + "59997 time (invisible): 220.11083984375 ms 10.0% \n", + "59998 render time (invisible): 72.118896484375 ms 10.0% \n", + "59999 time (full): 390.1201171875 ms 10.0% \n", + "\n", + " timestamp \n", + "0 2023-08-11 17:04:19.234 \n", + "1 2023-08-11 17:04:19.236 \n", + "2 2023-08-11 17:04:19.237 \n", + "3 2023-08-11 17:04:19.237 \n", + "4 2023-08-11 17:04:19.240 \n", + "... ... \n", + "59995 2023-08-12 00:45:01.208 \n", + "59996 2023-08-12 00:45:01.208 \n", + "59997 2023-08-12 00:45:01.428 \n", + "59998 2023-08-12 00:45:01.429 \n", + "59999 2023-08-12 00:45:01.429 \n", + "\n", + "[60000 rows x 3 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mask = df_prio['message'].str.contains('%')\n", + "percentage = df_prio[mask]\n", + "percentage\n", + "df_prio.loc[mask, 'percentage'] = percentage['message']\n", + "df_prio['percentage'].fillna(method='ffill', inplace=True)\n", + "df_prio.drop(percentage.index, inplace=True)\n", + "df_prio = df_prio.reset_index()\n", + "df_prio[['message', 'percentage','timestamp']]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "df['type']= df['message'].str.split().str[0]\n", + "df['update'] = df['message'].str.split(\"(\").str[1].str.split(\")\").str[0]\n", + "df['time (ms)'] = df['message'].str.split(\":\").str[1].str.split().str[0]\n", + "df['type'] = df['type'].apply(lambda x: x + ' time' if x == 'render' else x)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
time (ms)updatetype
01942.76708984375visibletime
1942.869140625visiblerender time
21.306884765625invisibletime
31945.81103515625fulltime
4Noneinvisiblerender time
............
59995168.304931640625visibletime
5999667.005126953125visiblerender time
59997220.11083984375invisibletime
5999872.118896484375invisiblerender time
59999390.1201171875fulltime
\n", + "

60000 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " time (ms) update type\n", + "0 1942.76708984375 visible time\n", + "1 942.869140625 visible render time\n", + "2 1.306884765625 invisible time\n", + "3 1945.81103515625 full time\n", + "4 None invisible render time\n", + "... ... ... ...\n", + "59995 168.304931640625 visible time\n", + "59996 67.005126953125 visible render time\n", + "59997 220.11083984375 invisible time\n", + "59998 72.118896484375 invisible render time\n", + "59999 390.1201171875 full time\n", + "\n", + "[60000 rows x 3 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_prio['type']= df_prio['message'].str.split().str[0]\n", + "df_prio['update'] = df_prio['message'].str.split(\"(\").str[1].str.split(\")\").str[0]\n", + "df_prio['time (ms)'] = df_prio['message'].str.split(\":\").str[1].str.split().str[0]\n", + "df_prio['type'] = df_prio['type'].apply(lambda x: x + ' time' if x == 'render' else x)\n", + "df_prio[['time (ms)', 'update','type']]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "from natsort import natsort_keygen\n", + "\n", + "# Convert \"time (ms)\" column to numeric data type\n", + "df['time (ms)'] = pd.to_numeric(df['time (ms)'], errors='coerce')\n", + "df['traces'] = pd.to_numeric(df['traces'])\n", + "\n", + "# aggregation_functions = {\n", + "# 'time (ms)': ['mean', lambda x: np.std(x, ddof=0) if len(x) > 1 else 0, 'var']\n", + "# }\n", + "aggregation_functions = {\n", + " 'time (ms)': ['mean', 'var']\n", + "}\n", + "sample_size = 25\n", + "\n", + "filtered_df = df.query('`time (ms)` == `time (ms)`')\n", + "filtered_df_visible = filtered_df[filtered_df['update']=='visible']\n", + "filtered_df_visible = filtered_df_visible[filtered_df_visible['type']=='time']\n", + "filtered_df_visible = filtered_df_visible[filtered_df_visible['percentage'] == \"100.0%\"]\n", + "\n", + "grouped_df = filtered_df_visible.groupby(['percentage', 'datapoints', 'samples', 'traces']).agg(aggregation_functions)\n", + "agg_df = grouped_df.reset_index()\n", + "\n", + "agg_df.columns = ['percentage', 'datapoints', 'samples', 'traces', 'mean_time', 'variance']\n", + "\n", + "agg_df['std_dev'] = np.sqrt(agg_df[\"variance\"])\n", + "agg_df['std_err'] = 1.96 * agg_df[\"std_dev\"]/np.sqrt(sample_size)\n", + "\n", + "agg_df = agg_df.sort_values(by=['datapoints','samples','traces'],key=natsort_keygen())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert \"time (ms)\" column to numeric data type\n", + "df_prio['time (ms)'] = pd.to_numeric(df_prio['time (ms)'], errors='coerce')\n", + "df_prio['traces'] = pd.to_numeric(df_prio['traces'])\n", + "\n", + "# aggregation_functions = {\n", + "# 'time (ms)': ['mean', lambda x: np.std(x, ddof=0) if len(x) > 1 else 0, 'var']\n", + "# }\n", + "aggregation_functions = {\n", + " 'time (ms)': ['mean', 'var']\n", + "}\n", + "\n", + "filtered_df_prio = df_prio.query('`time (ms)` == `time (ms)`')\n", + "filtered_df_prio_visible = filtered_df_prio[filtered_df_prio['update']=='visible']\n", + "filtered_df_prio_visible = filtered_df_prio_visible[filtered_df_prio_visible['type']=='time']\n", + "\n", + "grouped_df_prio = filtered_df_prio_visible.groupby(['percentage','datapoints', 'samples', 'traces']).agg(aggregation_functions)\n", + "agg_df_prio = grouped_df_prio.reset_index()\n", + "agg_df_prio.columns = ['percentage', 'datapoints', 'samples', 'traces', 'mean_time', 'variance']\n", + "agg_df_prio['std_dev'] = np.sqrt(agg_df_prio[\"variance\"])\n", + "agg_df_prio['std_err'] = 1.96 * agg_df_prio[\"std_dev\"]/np.sqrt(sample_size)\n", + "\n", + "\n", + "\n", + "agg_df_prio = agg_df_prio.sort_values(by=['datapoints','samples','traces','percentage'],key=natsort_keygen())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "agg_df['percentage'] = '100.0% (baseline)'\n", + "df_final = pd.concat([agg_df , agg_df_prio], ignore_index=True)\n", + "df_final = df_final.sort_values(by=['datapoints','samples','traces','percentage'],key=natsort_keygen())\n", + "df_final = df_final.reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "df_final.to_csv('../logs/final/processed_logs_final.csv')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "plotly-resampler-rFn5pKAA-py3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "2ee23f9e06e9276a9f67ea4ee15ee2ba149350665b01045ddc135410d3893be3" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/minimal_variable_threads.py b/tests/minimal_variable_threads.py new file mode 100644 index 00000000..c7fb4f4d --- /dev/null +++ b/tests/minimal_variable_threads.py @@ -0,0 +1,80 @@ +import argparse + +import numpy as np +import plotly.graph_objs as go + +from plotly_resampler.aggregation import EveryNthPoint + +# from dash import Input, Output, dcc, html +# from trace_updater import TraceUpdater +# import sys +# print(sys.path) +# sys.path.append('C:\\Users\\willi\\Documents\\ISIS\\Thesis\\plotly-resampler') +from plotly_resampler.figure_resampler import FigureResampler + +parser = argparse.ArgumentParser() +parser.add_argument("-n", "--npoints", type=int) +parser.add_argument("-s", "--nsamples", type=int) +parser.add_argument("-t", "--traces", type=int) + +args = parser.parse_args() +n = args.npoints +s = args.nsamples +t = args.traces + +# print(n) +# print(s) +# print(t) + + +# # Construct a high-frequency signal +# n=1_000_000 +# s=10_000 +# t=10 + + +def make_fig(n, s, t): + x = np.arange(n) + noisy_sin = (3 + np.sin(x / 200) + np.random.randn(len(x)) / 10) * x / (n / 10) + print(n / s) + # Construct the to-be resampled figure + fig = FigureResampler( + go.Figure(), + # show_mean_aggregation_size=False, + default_downsampler=EveryNthPoint(interleave_gaps=False), + default_n_shown_samples=s, + resampled_trace_prefix_suffix=("", ""), + ) + for i in range(t): + fig.add_trace( + go.Scattergl(name=f"sine-{i}", showlegend=True), + hf_x=x, + hf_y=noisy_sin + 10 * i, + ) + return fig + + +# Construct app & its layout +# app = dash.Dash(__name__) + +# app.layout = html.Div( +# [ +# dcc.Store(id="visible-indices", data={"visible": [], "invisible": []}), +# dcc.Graph(id="graph-id", figure=fig), +# TraceUpdater(id="trace-updater", gdID="graph-id",verbose=True), +# ] +# ) + +# n=1_000_000 +# s=4000 +# t=100 + +fig = make_fig(n, s, t) +# Register the callback + +fig.show_dash(mode="external", testing=True) +# # fig.register_update_graph_callback(app, "graph-id", "trace-updater", "visible-indices") + + +# if __name__ == "__main__": +# app.run_server(debug=True, port=8050) diff --git a/tests/test_visual_gain_threads.py b/tests/test_visual_gain_threads.py new file mode 100644 index 00000000..1470f9b4 --- /dev/null +++ b/tests/test_visual_gain_threads.py @@ -0,0 +1,158 @@ +import json +import os +import subprocess as sp +import time + +import numpy as np +from fr_selenium import FigureResamplerGUITests +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from seleniumwire import webdriver +from webdriver_manager.chrome import ChromeDriverManager, ChromeType + +# create a test for each value of n_traces, n_datapoints and shown_datapoints +# open new page +# loop over a range of percentages (% of shown traces) +# start timer (in front end via selenium? performance library js) +# apply 50% range zoom +# stop timer when visible update +# start another timer for invisible +# stop timer when invisible renders +# return to original range (may trigger timer in front end... prevent this!!) +# extract logs from this iteration into a file +# close page! + + +iterations = 1 # use to run this benchmarking process multiple times -> collect more data -> more accurate results + +percentages_hidden = np.array([0, 0.2, 0.5, 0.8, 0.9]) +n_traces = [ + 10, + 20, + 50, + 100 +] +n_datapoints = [ + 100_000, + 1_000_000, + 10_000_000 +] # hypothesis: this shouldn't affect the results too much? (if the biggest bottleneck is data transfer time) +n_shown_datapoints = [ + 100, + 1000, + 5000, + 10000 +] +for it in range(iterations): + print(f"iteration {it}") + options = Options() + # options.add_argument("--kiosk") #maximize window + d = DesiredCapabilities.CHROME + d["goog:loggingPrefs"] = {"browser": "ALL"} + driver = webdriver.Chrome( + ChromeDriverManager(chrome_type=ChromeType.GOOGLE).install(), + options=options, + desired_capabilities=d, + ) + + driver.maximize_window() + port = 8050 + fr = FigureResamplerGUITests(driver, port=port) + + try: + for t in n_traces: + for n in n_datapoints: + for s in n_shown_datapoints: + time.sleep(2) + proc = sp.Popen( + [ + "poetry", + "run", + "python", + "./tests/minimal_variable_threads.py", + "-n", + str(n), + "-s", + str(s), + "-t", + str(t), + ], + # creationflags=sp.CREATE_NEW_CONSOLE + ) + print(f"n_traces: {t}") + print(f"n_datapoints: {n}") + print(f"n_shown_datapoints: {s}") + print(f"iteration {it}") + + try: + time.sleep(20) + fr.go_to_page() + + time.sleep(1) + + # determine the number of traces that will be hidden corresponding to each percentage + n_traces_hidden = np.unique( + np.ceil(t * percentages_hidden) + ).astype(int) + # TODO: get final list of percentages (visible!) and print to console + + # print(n_traces_hidden) + last = t + for idx, j in enumerate(n_traces_hidden): + if idx == 0: + previous_n_hidden = 0 + else: + previous_n_hidden = n_traces_hidden[idx - 1] + # hide r traces from the last hidden trace + driver.execute_script(f'console.log("{100-((j/t)*100)}%")') + print(previous_n_hidden) + residual = n_traces_hidden[idx] - previous_n_hidden + print(residual) + residual_indices = [ + int(last - (i + 1)) for i in range(residual) + ] + last -= residual + if residual_indices != []: + fr.hide_legend_restyle(residual_indices) + + # after hiding the traces, (start the timer,) zoom in, then reset the axes for the next iteration + fr.drag_and_zoom( + "xy", x0=0.25, x1=0.75, y0=0.5, y1=0.5, testing=True + ) + + time.sleep(3) + fr.reset_axes(testing=True) + time.sleep(3) + with open( + f"./logs/n{n}_s{s}_t{t}_everynth_iter{it}.json", "w" + ) as logfile: + logfile.write(json.dumps(driver.get_log("browser"))) + print("done saving log") + except Exception as e: + raise e + finally: + print(proc.pid) + # p = ps.Process(proc.pid) + # print(f'pid {proc.pid}') + # print(f'process is running {p.is_running()}') + print(f"process is running {proc.poll is not None}") + + # proc.send_signal(signal.CTRL_C_EVENT) + + # this works with windows! add if clause for Linux version! (proc.kill works?) + os.system("TASKKILL /F /T /PID " + str(proc.pid)) + os.system( + "killport 8050 --view-only" + ) # requires pip install killport + # proc.kill() + print(f"process is running {proc.poll() is not None}") + + # os.kill(proc.pid, signal.SIGKILL) + + except Exception as ex: + raise ex + finally: + print("closing driver") + # driver.close() + print(driver is None) + driver.quit()