Merge pull request #826 from MetaCell/master

sync dev

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14 as jsbuild
+FROM node:18 as jsbuild
 
 ENV FOLDER=netpyne
 
@@ -11,6 +11,11 @@ RUN yarn install  --network-timeout 1000000000
 COPY webapp .
 RUN yarn build-dev
 
+### Download on a separate stage to run in parallel with buildkit
+FROM jupyter/base-notebook:hub-1.5.0 as downloads
+USER root
+RUN wget --no-check-certificate -O /nyhead.mat https://www.parralab.org/nyhead/sa_nyhead.mat
+
 ###
 FROM jupyter/base-notebook:hub-1.5.0
 ENV NB_UID=jovyan
@@ -31,22 +36,30 @@ COPY  --chown=1000:1000 requirements.txt requirements.txt
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
     pip install -r requirements.txt --prefer-binary
 
-COPY . .
-COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build
 
 # ToDo: fixme, for now remove the jupyter hub config json file because it overrides the default
 # and thus removes the frame ancestor cors settings
 RUN rm -f ~/.jupyter/*.json
 RUN chown $NB_UID .
 RUN chown $NB_UID /opt
 RUN rm -Rf workspace
-
-USER $NB_UID
-
 # sym link workspace pvc to $FOLDER
 RUN mkdir -p /opt/workspace
 RUN mkdir -p /opt/user
 
+COPY netpyne_ui netpyne_ui
+COPY utilities utilities
+COPY setup.py .
+COPY tests tests
+COPY NetPyNE-UI .
+COPY README.rst .
+COPY requirements-test.txt .
+
+
+USER $NB_UID
+
+
+
 
 ENV NEURON_HOME=/opt/conda
 
@@ -60,17 +73,22 @@ RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
 
 ARG BUILD_ARGS=""
 ARG WORKSPACE_VERSION=master
+
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
-  python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION
+  python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION --npm-skip
 
 
 RUN mv workspace /opt/workspace/tutorials
-RUN chown -R $NB_UID /opt/workspace/tutorials
+RUN chown -R $NB_UID /opt/workspace
 RUN ln -s /opt/workspace workspace
 
 RUN jupyter labextension disable @jupyterlab/hub-extension
-RUN wget --no-check-certificate -O $NP_LFPYKIT_HEAD_FILE https://www.parralab.org/nyhead/sa_nyhead.mat
 
+COPY --from=downloads --chown=1000:1000 /nyhead.mat $NP_LFPYKIT_HEAD_FILE
+COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build
+
+RUN chown -R $NB_UID /home/jovyan/.jupyter
+RUN touch app.log && chown $NB_UID app.log
 USER $NB_UID
 
 

netpyne_ui/helpers/neuroml.py

@@ -2,7 +2,9 @@
 import sys
 import logging
 from netpyne.specs import simConfig
+from packaging import version
 
+import pyneuroml
 from pyneuroml import pynml
 from pyneuroml.lems import generate_lems_file_for_neuroml
 from pyneuroml.pynml import read_neuroml2_file
@@ -13,57 +15,67 @@
 def convertLEMSSimulation(lemsFileName, compileMod=True):
     """Converts a LEMS Simulation file
 
-    Converts a LEMS Simulation file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html)
-    pointing to a NeuroML 2 file into the equivalent in NetPyNE
+    Converts a LEMS Simulation file
+    (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) pointing to a
+    NeuroML 2 file into the equivalent in NetPyNE
+
     Returns:
         simConfig, netParams for the model in NetPyNE
     """
     current_path = os.getcwd()
     try:
-        
+
         fullLemsFileName = os.path.abspath(lemsFileName)
-        tmp_path = os.path.dirname(fullLemsFileName) 
+        tmp_path = os.path.dirname(fullLemsFileName)
         if tmp_path:
             os.chdir(tmp_path)
         logging.info(
             "Importing LEMSSimulation with NeuroML 2 network from: %s"
             % fullLemsFileName
         )
 
-        result = pynml.run_lems_with_jneuroml_netpyne(
-            lemsFileName, only_generate_json=True, exit_on_fail=False)
-
-        if result == False:
-            raise Exception("Error loading lems file")
+        # feature to return output added in 1.0.9
+        if version.parse(pyneuroml.__version__) >= version.parse("1.0.9"):
+            result, output_msg = pynml.run_lems_with_jneuroml_netpyne(
+                lemsFileName, only_generate_json=True, exit_on_fail=False,
+                return_string=True, max_memory="1G")
+
+            if result is False:
+                raise Exception(f"Error loading lems file: {output_msg}")
+        else:
+            result = pynml.run_lems_with_jneuroml_netpyne(
+                lemsFileName, only_generate_json=True, exit_on_fail=False,
+                max_memory="1G")
+
+            if result is False:
+                raise Exception("Error loading lems file")
+
         lems = pynml.read_lems_file(lemsFileName)
 
         np_json_fname = fullLemsFileName.replace('.xml','_netpyne_data.json')
-    
+
         return np_json_fname
     finally:
         os.chdir(current_path)
 
 
-
-
 def convertNeuroML2(nml2FileName,  compileMod=True):
     """Loads a NeuroML 2 file into NetPyNE
     Loads a NeuroML 2 file into NetPyNE by creating a new LEMS Simulation
-    file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) and using jNeuroML
-    to convert it.
+    file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) and using
+    jNeuroML to convert it.
 
     Returns:
         simConfig, netParams for the model in NetPyNE
     """
     current_path = os.getcwd()
     try:
         fullNmlFileName = os.path.abspath(nml2FileName)
-        work_path = os.path.dirname(fullNmlFileName) 
+        work_path = os.path.dirname(fullNmlFileName)
         if not os.path.exists(work_path):
             os.makedirs(work_path)
         os.chdir(work_path)
         sys.path.append(work_path)
-
 
         logging.info(
                 "Importing NeuroML 2 network from: %s"

netpyne_ui/netpyne_geppetto.py

@@ -1001,6 +1001,9 @@ def getAvailableCellTypes(self):
         for p in self.netParams.cellParams:
             cell_types.add(p)
         return sorted(cell_types)
+
+    def getAvailableDensityTypes(self):
+        return ['uniform', '1DMap', '2DMap', 'distance']
 
     def getAvailableRxDSections(self, selectedRegion = None):
         sections = set([])

requirements.txt

@@ -29,11 +29,11 @@ jupyter-core==4.9.1
 jupyter-server==1.11.2
 jupyterhub==1.5.0
 jupyterlab==3.2.4
-neuromllite==0.5.1
+neuromllite==0.5.4
 jupyterthemes==0.20.0
 kiwisolver==1.2.0
 lesscpy==0.14.0
-libNeuroML==0.4.0
+libNeuroML==0.5.1
 lfpykit==0.5.1
 lxml==4.5.1
 Mako==1.1.0
@@ -44,7 +44,7 @@ mistune==0.8.4
 multimethod==1.3
 nbconvert==5.6.1
 nbformat==5.0.6
-netpyne==1.0.4.1
+netpyne==1.0.6
 NEURON==8.2.2
 numpy==1.18.5
 oauthlib==3.0.1
@@ -65,7 +65,7 @@ pycparser==2.20
 pyecore==0.11.7
 pygeppetto==0.8.1
 PyLEMS==0.5.9
-pyNeuroML==0.7.1
+pyNeuroML==1.0.10
 sentry_sdk==1.5.2
 dacite==1.6.0
 h5py==3.7.0

setup.py

@@ -18,7 +18,7 @@
 
 setuptools.setup(
     name="netpyne_ui",
-    version="1.0.0",
+    version="1.1.0",
     url="https://github.com/MetaCell/NetPyNE-UI",
     author="MetaCell",
     author_email="[email protected]",
@@ -42,9 +42,9 @@
     install_requires=[
         'jupyter-geppetto>=1.0.0',
         'NEURON>=8.2.2',
-        'netpyne>=1.0.4.1',
-        'neuromllite==0.5.1',
-        'pyNeuroML>=0.7.1',
+        'netpyne>=1.0.6',
+        'neuromllite==0.5.4',
+        'pyNeuroML>=1.0.10',
         'sentry_sdk>=1.5.2',
         'dacite>=1.6.0',
         'h5py>=3.7.0',

tests/frontend/e2e/tests/ControlPanel.test.js

@@ -30,7 +30,7 @@ const PASSWORD = 'testpassword'
 
 //TESTS:
 
-jest.setTimeout(300000);
+jest.setTimeout(3000000);
 let browser_control_panel;
 let control_panel_page;
 

tests/frontend/e2e/tests/EEG_and_Dipole_Tut#1.test.js

@@ -36,7 +36,7 @@ const PASSWORD = 'testpassword'
 
 //TESTS:
 
-jest.setTimeout(300000);
+jest.setTimeout(3000000);
 let browser_EEG_Dipole;
 let EEG_Dipole_page;
 
@@ -122,7 +122,7 @@ describe('EEG and Dipole Plot Test using Tutorial#1', () => {
 
         await console.log('Loading Tutorial #1')
         await EEG_Dipole_page.click(selectors.TUTORIAL_1_SELECTOR, { timeout: TIMEOUT })
-        await EEG_Dipole_page.waitForSelector(selectors.PYR_CELL_SELECTOR)
+        await EEG_Dipole_page.waitForSelector(selectors.PYR_CELL_SELECTOR, { timeout: TIMEOUT * 2 })
         await EEG_Dipole_page.waitForTimeout(PAGE_WAIT)
 
 
@@ -189,17 +189,20 @@ describe('EEG and Dipole Plot Test using Tutorial#1', () => {
     it('Dipole Plot', async () => {
 
         await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 2);
+        await EEG_Dipole_page.waitForSelector(selectors.DIPOLE_PLOT_SELECTOR)
         await EEG_Dipole_page.click(selectors.DIPOLE_PLOT_SELECTOR)
         await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
 
         await console.log('View Dipole Plot ...')
 
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 20);
-        await EEG_Dipole_page.click(selectors.CONNECTIONS_PLOT_SELECTOR, { timeout: TIMEOUT })
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT);
-        await EEG_Dipole_page.click(selectors.DIPOLE_PLOT_SELECTOR)
-        await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 3);
+        await EEG_Dipole_page.waitForSelector('#plot > div > div > img', { timeout: TIMEOUT * 10, hidden: false})
+
+        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 2);
+        // await EEG_Dipole_page.click(selectors.CONNECTIONS_PLOT_SELECTOR, { timeout: TIMEOUT })
+        // await EEG_Dipole_page.waitForTimeout(PAGE_WAIT);
+        // await EEG_Dipole_page.click(selectors.DIPOLE_PLOT_SELECTOR)
+        // await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
+        // await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 3);
 
         await console.log('... taking snapshot ...');
         expect(await EEG_Dipole_page.screenshot())
@@ -213,17 +216,20 @@ describe('EEG and Dipole Plot Test using Tutorial#1', () => {
     it('EEG Plot', async () => {
 
         await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 2);
+        await EEG_Dipole_page.waitForSelector(selectors.EEG_PLOT_SELECTOR)
         await EEG_Dipole_page.click(selectors.EEG_PLOT_SELECTOR)
-        await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
+        await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT, hidden: false})
 
         await console.log('View EEG Plot ...')
 
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 25);
-        await EEG_Dipole_page.click(selectors.CONNECTIONS_PLOT_SELECTOR, { timeout: TIMEOUT })
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 2);
-        await EEG_Dipole_page.click(selectors.EEG_PLOT_SELECTOR)
-        await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
-        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 3);
+        await EEG_Dipole_page.waitForSelector('#plot > div > div > img', { timeout: TIMEOUT * 10, hidden: false})
+
+        await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 7);
+        // await EEG_Dipole_page.click(selectors.CONNECTIONS_PLOT_SELECTOR, { timeout: TIMEOUT })
+        // await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 2);
+        // await EEG_Dipole_page.click(selectors.EEG_PLOT_SELECTOR)
+        // await EEG_Dipole_page.waitForSelector(selectors.CANVAS_SELECTOR, { timeout: TIMEOUT })
+        // await EEG_Dipole_page.waitForTimeout(PAGE_WAIT * 3);
 
         await console.log('... taking snapshot ...');
         expect(await EEG_Dipole_page.screenshot())

tests/frontend/e2e/tests/ExperimentManager_Tut#1.test.js

@@ -35,7 +35,7 @@ const PASSWORD = 'testpassword'
 
 //TESTS:
 
-jest.setTimeout(300000);
+jest.setTimeout(3000000);
 let browser_experiment_manager;
 let experiment_manager_page;
 
@@ -160,10 +160,11 @@ describe('Experiment Manager test using Tut#1', () => {
         await experiment_manager_page.click(selectors.CREATE_NEW_EXPERIMENT_SELECTOR)
 
         await experiment_manager_page.waitForSelector(selectors.CREATE_NEW_EXPERIMENT_POPUP_SELECTOR)
+        await experiment_manager_page.waitForSelector(selectors.CONFIRM_SELECTOR, { timeout: TIMEOUT, hidden: false })
         await experiment_manager_page.click(selectors.CONFIRM_SELECTOR)
-        await experiment_manager_page.waitForSelector(selectors.EXPERIMENT_NAME_SELECTOR, { timeout: TIMEOUT , visible: true})
+        await experiment_manager_page.waitForSelector(selectors.EXPERIMENT_NAME_SELECTOR, { timeout: TIMEOUT , hidden: false})
         await experiment_manager_page.waitForTimeout(PAGE_WAIT);
-        await expect(experiment_manager_page).toFill(selectors.EXPERIMENT_NAME_SELECTOR, 'Test Experiment')
+        await expect(experiment_manager_page).toFill(selectors.EXPERIMENT_NAME_SELECTOR, 'Test Experiment', { timeout: TIMEOUT, hidden: false })
         await experiment_manager_page.waitForTimeout(PAGE_WAIT);
         await experiment_manager_page.waitForSelector(selectors.PARAMETER_SELECTION_SELECTOR)
         await experiment_manager_page.click(selectors.PARAMETER_SELECTION_SELECTOR)