Refactor .travis.yml, cache dependencies

roboticslab-uc3m/questions-and-answers#48

.travis.yml

@@ -1,100 +1,127 @@
 language: cpp
 dist: trusty
 sudo: required
-
-os:
-  - linux
+os: linux
 
 compiler:
   - gcc
   - clang
 
+env:
+  global:
+    - CACHE_DIR=~/cached-deps
+  matrix:
+    - YARP_CHECKOUT=master
+    - YARP_CHECKOUT=v2.3.72.1
+    - YARP_CHECKOUT=v2.3.70.2
+
+matrix:
+  include:
+    - if: type = cron
+      compiler: gcc
+      env: ROBOTOLOGY_CHECKOUT=devel
+    - if: type = cron
+      compiler: clang
+      env: ROBOTOLOGY_CHECKOUT=devel
+
+cache:
+  directories: $CACHE_DIR
+
+addons:
+  apt:
+    packages:
+    - libopencv-dev
+
+    # YARP and KDL
+    - libeigen3-dev
+
+    # OpenRAVE
+    - libboost-all-dev
+    - libqt4-dev
+    - qt4-dev-tools
+    - libxml2-dev
+    - libode-dev
+    - libsoqt4-dev
+    - libcoin80-dev
+
+    # Aquila
+    - libsfml-dev
+
+    # unit testing
+    - libgtest-dev
+
+    # code coverage
+    - lcov
+
 before_install:
+  #-- Configure Git (needed by YCM)
   - if [ ! `git config --get user.email` ]; then `git config --global user.email '[email protected]'`; fi
   - if [ ! `git config --get user.name` ]; then `git config --global user.name 'Travis CI'`; fi
-  #-- Install yarp/kdl deps
-  - sudo apt-get install libeigen3-dev
-  #-- Install OpenRAVE deps
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install libboost-all-dev; fi
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install libqt4-dev qt4-dev-tools libxml2-dev libode-dev; fi # libfcl-dev not on Trusty used in this travis
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install libsoqt4-dev libcoin80-dev; fi
-  #-- Install Aquila deps
-  - sudo apt-get install libsfml-dev
-  #-- Install googletest
-  - sudo apt-get install libgtest-dev
 
-install:
-  #-- Install opencv
-  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install libopencv-dev; fi
-  #-- Install PCL
+  #-- Add source repository for PCL-related packages
   - sudo add-apt-repository --yes ppa:v-launchpad-jochen-sprickerhof-de/pcl
-  - sudo apt-get update -qq  # -qq No output except for errors
-  - sudo apt-get install libpcl-all libpcl-all-dev
+  - sudo apt-get -qq update
+
+  #-- Install PCL
+  - sudo apt-get install libpcl-all-dev
+
+  #-- Code coverage
+  - if [ "$CXX" = "g++" ]; then gem install coveralls-lcov; fi
+
+install:
   #-- Install ECF
-  - wget https://sourceforge.net/projects/ecfproject/files/ECF_1.4.1/ECF_1.4.1.tar.gz/download -O ECF_1.4.1.tar.gz
-  - tar -zxf ECF_1.4.1.tar.gz
-  - cd ECF_1.4.1
-  - wget http://ecf.zemris.fer.hr/ecf_install.sh
-  - echo "exit 0" > exit0.sh
-  - echo "yes" | sh ./ecf_install.sh; sh ./exit0.sh
-  - cd ..
-  #-- Install cmake
-  - sudo apt-get install cmake
-  #-- Install yarp
-  - git clone https://github.com/robotology/yarp
-  - cd yarp && mkdir build && cd build
-  - cmake .. -DSKIP_ACE=ON
-  - make
-  - sudo make install
-  - cd ../..
+  - wget https://sourceforge.net/projects/ecfproject/files/ECF_1.4.1/ECF_1.4.1.tar.gz/download -O ~/ECF_1.4.1.tar.gz
+  - tar -zxf ~/ECF_1.4.1.tar.gz -C ~
+  - wget http://ecf.zemris.fer.hr/ecf_install.sh -P ~/ECF_1.4.1
+  - echo "exit 0" > ~/ECF_1.4.1/exit0.sh
+  - echo "yes" | sh ~/ECF_1.4.1/ecf_install.sh; sh ~/ECF_1.4.1/exit0.sh
+
+  #-- Install YARP
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name YARP
+    --repo-url https://github.com/robotology/yarp
+    --repo-checkout "${ROBOTOLOGY_CHECKOUT:-$YARP_CHECKOUT}"
+    --additional-cmake-options "-DSKIP_ACE:BOOL=ON"
+    --prepend-to-linker-path lib
+    --additional-export-paths "YARP_DATA_DIRS;share/yarp"
+
   #-- Install OpenRAVE
-  - git clone --branch latest_stable https://github.com/rdiankov/openrave.git
-  - cd openrave && mkdir build && cd build
-  - cmake .. -DOPENRAVE_PLUGIN_FCLRAVE=OFF
-  - make
-  - sudo make install
-  - cd ../..
-  #-- Install kdl
-  - git clone https://github.com/orocos/orocos_kinematics_dynamics
-  - cd orocos_kinematics_dynamics/orocos_kdl && mkdir build && cd build
-  - cmake ..
-  - make
-  - sudo make install
-  - cd ../../..
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name OpenRAVE
+    --repo-url https://github.com/rdiankov/openrave
+    --repo-checkout master
+    --prepend-to-linker-path lib
+
+  #-- Install KDL
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name KDL
+    --repo-url https://github.com/orocos/orocos_kinematics_dynamics
+    --repo-checkout master
+    --cmake-home-dir orocos_kdl
+    --prepend-to-linker-path lib
+
   #-- Install Aquila
-  - git clone git://github.com/zsiciarz/aquila.git
-  - cd aquila; mkdir build; cd build
-  - cmake ..
-  - make
-  - sudo make install
-  - cd ../..
-  # see http://gronlier.fr/blog/2015/01/adding-code-coverage-to-your-c-project/
-  - if [ "$CXX" = "g++" ]; then wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.11.orig.tar.gz; fi
-  - if [ "$CXX" = "g++" ]; then tar xf lcov_1.11.orig.tar.gz; fi
-  - if [ "$CXX" = "g++" ]; then sudo make -C lcov-1.11/ install; fi
-  - if [ "$CXX" = "g++" ]; then gem install coveralls-lcov; fi
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name OpenRAVE
+    --repo-url https://github.com/zsiciarz/aquila
+    --repo-checkout master
+    --prepend-to-linker-path lib
 
 before_script:
-  - mkdir -p build && cd build
+  - mkdir -p "$TRAVIS_BUILD_DIR/build" && cd "$_"
+  - cmake .. -DCMAKE_INSTALL_PREFIX="$PWD/install" -DENABLE_coverage:BOOL=ON
+  - make install
+  - export LD_LIBRARY_PATH=$PWD/install/lib:$LD_LIBRARY_PATH
 
 script:
-  - cmake .. -DENABLE_coverage=ON
-  - make
-  - sudo make install
-  - sudo ldconfig
-  #- bin/testKdlSolver
+  - echo "No unit tests available"
 
 after_success:
-  #  - cd ${TRAVIS_BUILD_DIR}
-  # capture coverage info
-  - if [ "$CXX" = "g++" ]; then lcov --directory . --capture --output-file coverage.info; fi
-  # filter out system and test code
-  - if [ "$CXX" = "g++" ]; then lcov --remove coverage.info  '/usr/*' 'test/*' --output-file coverage.info; fi
-  # debug before upload
-  - if [ "$CXX" = "g++" ]; then lcov --list coverage.info; fi
-  # uploads to coveralls
-  #- if [ "$CXX" = "g++" ]; then coveralls-lcov --source-encoding=ISO-8859-1 --repo-token InWqFy6VWMrEUzr2Rn8XV0cGqv3wXWo2d coverage.info; fi
-  - if [ "$CXX" = "g++" ]; then coveralls-lcov --source-encoding=ISO-8859-1 coverage.info; fi
-
-  - sudo make uninstall
+  #-- Code coverage
+  - if [ "$CXX" = "g++" ]; then source "$TRAVIS_BUILD_DIR/scripts/travis/coverage.sh"; fi
 
+  - make uninstall

scripts/travis/coverage.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set +e
+
+# see http://gronlier.fr/blog/2015/01/adding-code-coverage-to-your-c-project/
+
+#-- Capture coverage info
+lcov --directory . --capture --output-file coverage.info
+
+#-- Filter out system and test code
+lcov --remove coverage.info  '/usr/*' 'tests/*' --output-file coverage.info
+
+#-- Debug before upload
+lcov --list coverage.info
+
+#-- Upload to coveralls
+coveralls-lcov --source-encoding=ISO-8859-1 coverage.info

scripts/travis/git-cache-dependency.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+set -e
+
+#-- Initialize variables
+package_name=
+repo_url=
+repo_checkout=
+clone_only_branches="master devel develop production"
+cmake_home_dir=
+additional_cmake_options=
+prepend_to_linker_path=
+prepend_to_standard_path=
+additional_export_paths=
+
+#-- Available getopt long option names
+#-- https://gist.github.com/magnetikonline/22c1eb412daa350eeceee76c97519da8
+ARGUMENT_LIST=(
+    "package-name"
+    "repo-url"
+    "repo-checkout"
+    "clone-only-branches"
+    "cmake-home-dir"
+    "additional-cmake-options"
+    "prepend-to-linker-path"
+    "prepend-to-standard-path"
+    "additional-export-paths"
+)
+
+#-- Read arguments
+opts=$(getopt \
+    --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")" \
+    --name "$(basename "${BASH_SOURCE[0]}")" \
+    --options "" \
+    -- "$@"
+)
+
+eval set -- $opts
+
+#-- Parse options
+while [ "$#" -gt 1 ]; do
+    in=$(echo "$1" | sed -e 's/^--//')
+    for v in "${ARGUMENT_LIST[@]}"; do
+        if [ "$v" = "$in" ]; then
+            declare "$(echo "$v" | tr '-' '_')"="$2"
+            break
+        fi
+    done
+    shift 2
+done
+
+#-- Check required arguments
+if [ -z "$package_name" ] || [ -z "$repo_url" ] || [ -z "$repo_checkout" ]; then
+    echo "Missing required options. Traceback:"
+    for v in "${ARGUMENT_LIST[@]}"; do
+        v_param=$(echo $v | tr '-' '_')
+        echo "  --$v=${!v_param}"
+    done
+    return 1
+fi
+
+#-- Configure paths
+repo_source_dir=~/"$package_name-$repo_checkout"
+repo_build_dir="$repo_source_dir/build"
+repo_cache_dir="$CACHE_DIR/$package_name-$repo_checkout"
+cmake_home_dir="$repo_source_dir/$cmake_home_dir"
+
+#-- Configure CMake command line options
+repo_cmake_options="$additional_cmake_options -DCMAKE_INSTALL_PREFIX:PATH=$repo_cache_dir"
+
+is_clone_only_branch=false
+
+for branch in $clone_only_branches; do
+    if [ "$repo_checkout" = "$branch" ]; then
+        is_clone_only_branch=true
+        break
+    fi
+done
+
+if $is_clone_only_branch; then
+
+    #-- Clone, build and store in cache
+
+    echo "Cloning $package_name's $repo_checkout branch"
+    git clone --depth=1 --branch="$repo_checkout" "$repo_url" "$repo_source_dir"
+    last_commit_sha=$(git -C "$repo_source_dir" rev-parse HEAD)
+
+    if [ ! -d "$repo_cache_dir" ] || \
+       [ ! -f "$repo_cache_dir/.last_commit_sha" ] || \
+       [ ! "$(cat $repo_cache_dir/.last_commit_sha)" = "$last_commit_sha" ];
+    then
+        echo "$package_name not in cache or not the latest commit of $repo_checkout branch"
+        rm -rf "$repo_cache_dir"/*
+        cmake -H"$cmake_home_dir" -B"$repo_build_dir" $repo_cmake_options
+        make -C "$repo_build_dir" -j$(nproc) install
+        echo "$last_commit_sha" > "$repo_cache_dir/.last_commit_sha"
+    else
+        echo "$package_name found in cache ($(cat $repo_cache_dir/.last_commit_sha))"
+    fi
+
+else
+
+    #-- Download zipped file from archive, build and store in cache
+
+    if [ ! -d "$repo_cache_dir" ]; then
+        echo "Downloading $package_name $repo_checkout from archive"
+        wget -q "$repo_url/archive/$repo_checkout.tar.gz" -P "$repo_source_dir"
+        tar xzf "$repo_source_dir/$repo_checkout.tar.gz" -C "$repo_source_dir" --strip-components=1
+        cmake -H"$cmake_home_dir" -B"$repo_build_dir" $repo_cmake_options
+        make -C "$repo_build_dir" -j$(nproc) install
+    else
+        echo "$package_name $repo_checkout already in cache"
+    fi
+
+fi
+
+#-- Make installed package discoverable by CMake's find_package() command
+export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH:$repo_cache_dir"
+
+#-- Miscellanea, prepends stuff to LD_LIBRARY_PATH and PATH
+
+if [ ! -z "$prepend_to_linker_path" ]; then
+    export LD_LIBRARY_PATH="$repo_cache_dir/$prepend_to_linker_path:$LD_LIBRARY_PATH"
+fi
+
+if [ ! -z "$prepend_to_standard_path" ]; then
+    export PATH="$repo_cache_dir/$prepend_to_standard_path:$PATH"
+fi
+
+#-- Expands lists of values of the form 'VAR1;val1;VAR2;val2;...' to:
+#--  export VAR1=val1
+#--  export VAR2=val2
+#--  ...
+if [ ! -z "$additional_export_paths" ]; then
+    IFS=';' read -ra ITEMS <<< "$additional_export_paths"
+    for (( count=0; count<"${#ITEMS[@]}"; count+=2 )); do
+        export "${ITEMS[$count]}"="$repo_cache_dir/${ITEMS[$(( $count+1 ))]}:${!ITEMS[$count]}"
+    done
+fi