Feature/hierarchical decoding models

.flake8

@@ -0,0 +1,18 @@
+[flake8]
+max-line-length = 99
+ignore =
+    W504,
+    W503,
+    W605,  # invalid escape sequence '\ '
+    E266,
+    E402,  # module level import not at top of file
+    E226,  # missing whitespace around arithmetic operator
+exclude =
+    .git,
+    __pycache__,
+    __init__.py,
+    build,
+    dist,
+    docs/*
+    example/*
+    scratch/*

CONTRIBUTING.md

@@ -0,0 +1,79 @@
+# How to contribute
+
+If you're interested in contributing to the behavenet package, please contact the project 
+developer Matt Whiteway at m.whiteway ( at ) columbia.edu.
+
+If you would like to add a new Pytorch model to the package, you can find more detailed information
+[here](behavenet/models/README.md).
+
+Before submitting a pull request, please follow these steps:
+
+## Style
+
+The behavenet package follows the PEP8 style guidelines, and allows for line lengths of up to 99 
+characters. To ensure that your code matches these guidelines, please flake your code using the 
+provided configuration file. You will need to first install flake8 in the behavenet conda 
+environment:
+
+```bash
+(behavenet) $: pip install flake8
+```
+
+Once all code, tests, and documentation are in place, you can run the flaker from from the project
+directory:
+
+```bash
+(behavenet) $: flake8
+```
+
+## Documentation
+
+Behavenet uses Sphinx and readthedocs to provide documentation to developers and users. 
+
+* complete all docstrings in new functions using google's style (see source code for examples)
+* provide inline comments when necessary; the more the merrier
+* add a new user guide if necessary (`docs/source/user_guide.[new_model].rst`)
+* update data structure docs if adding to hdf5 (`docs/source/data_structure.rst`)
+* add new hyperparams to glossary (`docs/source/glossary.rst`)
+
+To check the documentation, you can compile it on your local machine first. To do so you will need
+to first install sphinx in the behavenet conda environment:
+
+```bash
+(behavenet) $: pip install sphinx==3.2.0 sphinx_rtd_theme==0.4.3 sphinx-automodapi==0.12 
+```
+
+To compile the documentation, from the behavenet project directory cd to `behavenet/docs` and run 
+the make file:
+
+```bash
+(behavenet) $: cd docs
+(behavenet) $: make html 
+```
+
+## Testing
+
+Behavenet uses pytest to unit test the package; in addition, there is an integration script 
+provided to ensure the interlocking pieces play nicely. Please write unit tests for all new 
+(non-plotting) functions, and if you updated any existing functions please update the corresponding
+unit tests.
+
+To run the unit tests, first install pytest in the behavenet conda environment:
+
+```bash
+(behavenet) $: pip install pytest
+```
+
+Then, from the project directory, run:
+
+```bash
+(behavenet) $: pytest
+```
+
+To run the integration script:
+
+```bash
+(behavenet) $: python tests/integration.py
+```
+
+Running the integration test will take approximately 1 minute with a GPU.

README.md

@@ -1,7 +1,5 @@
 # BehaveNet
 
-NOTE: This is a beta version, we will release the first stable version by early February.
-
 BehaveNet is a probabilistic framework for the analysis of behavioral video and neural activity. 
 This framework provides tools for compression, segmentation, generation, and decoding of behavioral 
 videos. Please see the 
@@ -12,7 +10,7 @@ for more information about how to install the software and begin fitting models
 
 Additionally, we provide an example dataset and several jupyter notebooks that walk you through how 
 to download the dataset, fit models, and analyze the results. The jupyter notebooks can be found 
-[here](example).
+[here](examples).
 
 ## Bibtex
 

behavenet/data/__init__.py

@@ -1 +1 @@
-"""Test string"""
+"""Data module"""

behavenet/data/data_generator.py

@@ -192,14 +192,14 @@ def __init__(
         self.n_trials = None
         for i, signal in enumerate(signals):
             if signal == 'images' or signal == 'neural' or signal == 'labels' or \
-                    signal == 'labels_sc':
+                    signal == 'labels_sc' or signal == 'labels_masks':
                 data_file = paths[i]
                 with h5py.File(data_file, 'r', libver='latest', swmr=True) as f:
                     self.n_trials = len(f[signal])
                     break
             elif signal == 'ae_latents':
                 try:
-                    latents = _load_pkl_dict(self.paths[signal], 'latents') #[0]
+                    latents = _load_pkl_dict(self.paths[signal], 'latents')
                 except FileNotFoundError:
                     raise NotImplementedError(
                         ('Could not open %s\nMust create ae latents from model;' +
@@ -274,7 +274,8 @@ def __getitem__(self, idx):
                     else:
                         sample[signal] = f[signal][str('trial_%04i' % idx)][()].astype(dtype)
 
-            elif signal == 'neural' or signal == 'labels' or signal == 'labels_sc':
+            elif signal == 'neural' or signal == 'labels' or signal == 'labels_sc' \
+                    or signal == 'labels_masks':
                 dtype = 'float32'
                 with h5py.File(self.paths[signal], 'r', libver='latest', swmr=True) as f:
                     if idx is None:
@@ -286,25 +287,21 @@ def __getitem__(self, idx):
                     else:
                         sample[signal] = [f[signal][str('trial_%04i' % idx)][()].astype(dtype)]
 
-            elif signal == 'ae_latents':
+            elif signal == 'ae_latents' or signal == 'latents':
                 dtype = 'float32'
-                sample[signal] = self._try_to_load(
-                    signal, key='latents', idx=idx, dtype=dtype)
+                sample[signal] = self._try_to_load(signal, key='latents', idx=idx, dtype=dtype)
 
             elif signal == 'ae_predictions':
                 dtype = 'float32'
-                sample[signal] = self._try_to_load(
-                    signal, key='predictions', idx=idx, dtype=dtype)
+                sample[signal] = self._try_to_load(signal, key='predictions', idx=idx, dtype=dtype)
 
             elif signal == 'arhmm' or signal == 'arhmm_states':
                 dtype = 'int32'
-                sample[signal] = self._try_to_load(
-                    signal, key='states', idx=idx, dtype=dtype)
+                sample[signal] = self._try_to_load(signal, key='states', idx=idx, dtype=dtype)
 
             elif signal == 'arhmm_predictions':
                 dtype = 'float32'
-                sample[signal] = self._try_to_load(
-                    signal, key='predictions', idx=idx, dtype=dtype)
+                sample[signal] = self._try_to_load(signal, key='predictions', idx=idx, dtype=dtype)
 
             else:
                 raise ValueError('"%s" is an invalid signal type' % signal)
@@ -626,7 +623,7 @@ def next_batch(self, dtype):
 
         if self.as_numpy:
             for i, signal in enumerate(sample):
-                if signal is not 'batch_idx':
+                if signal != 'batch_idx':
                     sample[signal] = [ss.cpu().detach().numpy() for ss in sample[signal]]
         else:
             if self.device == 'cuda':