diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c3b951d..9fe99fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,13 @@ on: branches: - master - release + pull_request: + branches: + - master jobs: build: - name: CI + name: CI (tests) runs-on: ubuntu-latest steps: - name: Checkout trigger commit @@ -37,35 +40,35 @@ jobs: - name: Transfer source files to CI-server if: success() run: | - scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} -r ../DeerAnalysis2 ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ - ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} scp -r /home/lufa/scp_traffic/DeerAnalysis2 ${{ secrets.CISERVER_HOST }}:/home/deeranalysis/ - ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f -r /home/lufa/scp_traffic/DeerAnalysis2" + scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} -r ../DeerLab ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} scp -r /home/lufa/scp_traffic/DeerLab ${{ secrets.CISERVER_HOST }}:~ + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f -r /home/lufa/scp_traffic/DeerLab" - name: Run test suite connected to CI-Server if: success() run: | ssh -tt -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} -t ${{ secrets.GATEWAY_HOST }} ssh ${{ secrets.CISERVER_HOST }} << EOF #Run test suite - cd DeerAnalysis2/build - matlab -nodesktop -r "datestsuite;exit" + cd DeerLab/build + matlab -nodesktop -r "dltestsuite;exit" python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "coverage_badge.json" --bucket deershields python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "testsuite_badge.json" --bucket deershields #Transfer log file to the gateway - scp -o StrictHostKeyChecking=no -i ~/.ssh/gateway_key -P ${{ secrets.GATEWAY_PORT }} ./datestsuite.error ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ || true + scp -o StrictHostKeyChecking=no -i ~/.ssh/gateway_key -P ${{ secrets.GATEWAY_PORT }} ./dltestsuite.error ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ || true #Delete the source files - cd /home/deeranalysis/ - rm -f -r DeerAnalysis2 + cd ~ + rm -f -r DeerLab EOF - name: Disconnect CI-server if: success() run: | #Transfer log file to GitHub VM - scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/datestsuite.error ~/ || true - ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f /home/lufa/scp_traffic/datestsuite.error" || true + scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/dltestsuite.error ~/ || true + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f /home/lufa/scp_traffic/dltestsuite.error" || true rm -f ~/sshkey - name: All tests passed successfully if: success() run: | - ! [ -f ~/datestsuite.error ] && echo "Test suite passed" || (echo "Test suite failed" && deerfailed) + ! [ -f ~/dltestsuite.error ] && echo "Test suite passed" || (echo "Test suite failed" && deerfailed) - name: Disconnect VPN connection if: always() run: | diff --git a/.github/workflows/ci_dev.yml b/.github/workflows/ci_dev.yml new file mode 100644 index 00000000..c20a6cf3 --- /dev/null +++ b/.github/workflows/ci_dev.yml @@ -0,0 +1,77 @@ +name: Scheduled Continuous Integration + +on: + schedule: + # Run every day at 6:30 AM + - cron: '30 6 * * *' + +jobs: + build: + name: Scheduled CI (dev) + runs-on: ubuntu-latest + steps: + - name: Checkout trigger commit + uses: actions/checkout@v2 + with: + ref: develop + - name: Install dependencies + run: | + sudo apt-get install -y openvpn openconnect + - name: Prepare VM + if: success() + run: | + rm -f -r ./.git + echo "${{ secrets.GATEWAY_KEY }}" > ~/sshkey + sudo chmod 600 ~/sshkey + - name: Establish VPN connection + if: success() + run: | + # Force returning packets to be routed over public interface and not over the VPN + sudo ip rule add from $(ip route get 1 | grep -Po '(?<=src )(\S+)') table 128 + sudo ip route add table 128 to $(ip route get 1 | grep -Po '(?<=src )(\S+)')/32 dev $(ip -4 route ls | grep default | grep -Po '(?<=dev )(\S+)') + sudo ip route add table 128 default via $(ip -4 route ls | grep default | grep -Po '(?<=via )(\S+)') + # Start network tunnel (TUN) device + sudo openvpn --mktun --dev tun1 + sudo ifconfig tun1 up + # Launch VPN connection + echo "${{ secrets.VPN_PASSWD }}" | sudo openconnect ${{ secrets.VPN_SERVER }} --user=${{ secrets.VPN_USER }} --background --passwd-on-stdin + - name: Transfer source files to CI-server + if: success() + run: | + scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} -r ../DeerLab ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} scp -r /home/lufa/scp_traffic/DeerLab ${{ secrets.CISERVER_HOST }}:~ + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f -r /home/lufa/scp_traffic/DeerLab" + - name: Run test suite connected to CI-Server + if: success() + run: | + ssh -tt -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} -t ${{ secrets.GATEWAY_HOST }} ssh ${{ secrets.CISERVER_HOST }} << EOF + #Run test suite + cd DeerLab/build + matlab -nodesktop -r "dltestsuite;exit" + python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "coverage_badge.json" --bucket deershields + python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "testsuite_badge.json" --bucket deershields + #Transfer log file to the gateway + scp -o StrictHostKeyChecking=no -i ~/.ssh/gateway_key -P ${{ secrets.GATEWAY_PORT }} ./dltestsuite.error ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ || true + #Delete the source files + cd ~ + rm -f -r DeerLab + EOF + - name: Disconnect CI-server + if: success() + run: | + #Transfer log file to GitHub VM + scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/dltestsuite.error ~/ || true + ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f /home/lufa/scp_traffic/dltestsuite.error" || true + rm -f ~/sshkey + - name: All tests passed successfully + if: success() + run: | + ! [ -f ~/dltestsuite.error ] && echo "Test suite passed" || (echo "Test suite failed" && deerfailed) + - name: Disconnect VPN connection + if: always() + run: | + #Ensure the VPN is disconnected by killing openconnect + sudo pkill openconnect + #Remove TUN device + sudo ifconfig tun1 down + sudo openvpn --rmtun --dev tun1 \ No newline at end of file diff --git a/.github/workflows/docbuilder.yml b/.github/workflows/deploy_ghpages.yml similarity index 62% rename from .github/workflows/docbuilder.yml rename to .github/workflows/deploy_ghpages.yml index 7c573553..6aa5d236 100644 --- a/.github/workflows/docbuilder.yml +++ b/.github/workflows/deploy_ghpages.yml @@ -1,41 +1,47 @@ -name: Webpage update - -on: - push: - branches: - - master - paths: - - 'docsrc/**' - - '.github/**' - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install sphinx==1.8.0 - python -m pip install sphinx_rtd_theme - python -m pip install sphinxcontrib-matlabdomain - python -m pip install sphinxcontrib-httpdomain - python -m pip install boto3 - sudo apt install texlive-extra-utils - sudo apt-get install texlive-latex-extra - - - name: Run Sphinx Builder - run: | - cd docsrc - sphinx-build -E -b html ./source ../docs - - name: Connect and update DeerAnalysis AWS S3 Bucket - run: | - cd build - python -m synchS3 -k ${{ secrets.S3_ACCESS }} -s ${{ secrets.S3_SECRETACCESS }} -d "../docs" --bucket deeranalysis.org \ No newline at end of file +name: Docs Build & Deployment + +on: + push: + branches: + - master + - hotfixes + paths: + - 'build/**' + - 'docsrc/**' + + schedule: + # Run once a week on Sunday at 12:00 PM + - cron: '0 12 * * 0' + +jobs: + + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install sphinx==1.8.0 + python -m pip install sphinx_rtd_theme + python -m pip install sphinxcontrib-matlabdomain + python -m pip install sphinxcontrib-httpdomain + sudo apt install texlive-extra-utils + sudo apt-get install texlive-latex-extra + + - name: Build multi-version docs + run: | + cd build + python -m multiversedocs + cd .. + + - name: Deploy to GH-Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./multidocs \ No newline at end of file diff --git a/.gitignore b/.gitignore index 114c7ba3..eec3968d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ docsrc/aws_access_keys.txt docsrc/__pycache__/ build/.ssh/ tutorials/**/*.pdf -tutorials/**/*.m +tutorials/**/*.mlx tests/_tutorials -tests/_tutorials/** \ No newline at end of file +tests/_tutorials/** +build/__pycache__/multiversedocs.cpython-37.pyc +multidocs/**/* +build/__pycache__/multiversedocs_testing.cpython-37.pyc diff --git a/README.md b/README.md index c59f8932..0c860c11 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,28 @@
- +
- - - - + + +
### About -The DeerLab software package is a MATLAB toolbox for the analysis of data from DEER (double electron-electron resonance) spectroscopy and similar dipolar spectroscopy techniques (DQC, RIDME, SIFTER). The main homepage can be found at www.deeranalysis.org. This is the GitHub repository of the DeerLab source code, including instructions for compiling and installing DeerLab. +The DeerLab software package is a MATLAB toolbox for the analysis of data from DEER (double electron-electron resonance) spectroscopy and similar dipolar EPR spectroscopy techniques (RIDME, DQC, SIFTER). The main homepage can be found at [jeschkelab.github.io/DeerLab](https://jeschkelab.github.io/DeerLab/). This is the GitHub repository of the DeerLab source code, including instructions for compiling and installing DeerLab. -It consists of a collection of functions that perform single processing or fitting tasks. They can be combined in scripts to generate custom analysis workflows. +It consists of a collection of functions that perform modelling, processing or fitting tasks. They can be combined in scripts to build custom data analysis workflows. -To model distance distributions, DeerLab supports two types of model classes and associated workflows: parameter-free models (as used in Tikhonov regularization) as well as a series of parameterized models (mutli-Gaussians etc). It also provides a selection of background models. There are functions for generating synthetic datasets as well as for fitting and analyzing experimental data sets. +To model distance distributions, DeerLab supports two types of model classes and associated workflows: parameter-free models (as used in Tikhonov regularization) as well as a series of parameterized models (mutli-Gaussians etc). It also provides a selection of background and experiment models. There are functions for generating synthetic datasets as well as for fitting and analyzing experimental data sets. ### Requirements -The application programming interface (API) of DeerLab requires the following products: +DeerLab requires the following products: - * MATLAB (oldest version supported R2016b) (see