Refactor .travis.yml, cache dependencies

roboticslab-uc3m/questions-and-answers#48

.travis.yml

@@ -7,85 +7,101 @@ compiler:
   - gcc
   - clang
 
+env:
+  global:
+    - CACHE_DIR=~/cached-deps
+  matrix:
+    - YARP_CHECKOUT=master
+    - YARP_CHECKOUT=v2.3.72.1
+    - YARP_CHECKOUT=v2.3.70.2
+
+matrix:
+  include:
+    - if: type = cron
+      compiler: gcc
+      env: ROBOTOLOGY_CHECKOUT=devel
+    - if: type = cron
+      compiler: clang
+      env: ROBOTOLOGY_CHECKOUT=devel
+
+cache:
+  directories: $CACHE_DIR
+
+addons:
+  apt:
+    packages:
+    - libopencv-dev
+    - libusb-1.0-0-dev # OpenNI
+    - libudev-dev # OpenNI
+    - openjdk-6-jdk # OpenNI
+    - freeglut3-dev # OpenNI
+
 before_install:
-  - sudo add-apt-repository --yes ppa:v-launchpad-jochen-sprickerhof-de/pcl  # for PCL-related packages
-  - sudo apt-get -qq update
-  - '[[ "$TRAVIS_EVENT_TYPE" = cron ]] && export ROBOTOLOGY_CHECKOUT=devel || export ROBOTOLOGY_CHECKOUT=master'
+  #-- Configure Git (needed by YCM)
+  - if [ ! `git config --get user.email` ]; then `git config --global user.email '[email protected]'`; fi
+  - if [ ! `git config --get user.name` ]; then `git config --global user.name 'Travis CI'`; fi
 
-  #-- Install OpenCV
-  - sudo apt-get install libopencv-dev
+  #-- Add source repository for PCL-related packages
+  - sudo add-apt-repository --yes ppa:v-launchpad-jochen-sprickerhof-de/pcl
+  - sudo apt-get -qq update
 
   #-- Install PCL
   - sudo apt-get install libpcl-all libpcl-all-dev
 
-  #-- Install OpenNI deps
-  - sudo apt-get install git libusb-1.0-0-dev libudev-dev
-  - sudo apt-get install openjdk-6-jdk  # if not using other java version
-  - sudo apt-get install freeglut3-dev
+  #-- Code coverage
+  - if [ "$CXX" = "g++" ]; then gem install coveralls-lcov; fi
 
 install:
-  # Install color-debug
-  - git clone https://github.com/roboticslab-uc3m/color-debug
-  - cd color-debug && mkdir build && cd build
-  - cmake ..
-  - sudo make install
-  - cd ../..
-
   #-- Install OpenNI2
-  - git clone https://github.com/occipital/OpenNI2.git
-  - cd OpenNI2
-  - make -j$(nproc)
-  - sudo ln -s $PWD/Bin/x64-Release/libOpenNI2.so /usr/local/lib/
-  - sudo ln -s $PWD/Bin/x64-Release/OpenNI2/ /usr/local/lib/
-  - sudo ln -s $PWD/Include /usr/local/include/OpenNI2
+  - git clone https://github.com/occipital/OpenNI2.git ~/OpenNI2
+  - make -C ~/OpenNI2 -j$(nproc)
+  - sudo ln -s ~/OpenNI2/Bin/x64-Release/libOpenNI2.so /usr/local/lib/
+  - sudo ln -s ~/OpenNI2/Bin/x64-Release/OpenNI2/ /usr/local/lib/
+  - sudo ln -s ~/OpenNI2/Include /usr/local/include/OpenNI2
   - sudo ldconfig
-  - cd ..
 
   #-- Install NiTE
-  - wget https://sourceforge.net/projects/roboticslab/files/External/nite/NiTE-Linux-x64-2.2.tar.bz2
-  - tar xvf NiTE-Linux-x64-2.2.tar.bz2
-  - sudo ln -s $PWD/NiTE-Linux-x64-2.2/Redist/libNiTE2.so /usr/local/lib/
-  - sudo ln -s $PWD/NiTE-Linux-x64-2.2/Include /usr/local/include/NiTE-Linux-x64-2.2
+  - wget https://sourceforge.net/projects/roboticslab/files/External/nite/NiTE-Linux-x64-2.2.tar.bz2 ~
+  - tar xjf ~/NiTE-Linux-x64-2.2.tar.bz2
+  - sudo ln -s ~/NiTE-Linux-x64-2.2/Redist/libNiTE2.so /usr/local/lib/
+  - sudo ln -s ~/NiTE-Linux-x64-2.2/Include /usr/local/include/NiTE-Linux-x64-2.2
 
   #-- Install YCM
-  - git clone --branch="$ROBOTOLOGY_CHECKOUT" https://github.com/robotology/ycm
-  - cd ycm && mkdir build && cd build
-  - cmake ..
-  - make -j$(nproc)
-  - sudo make install
-  - cd ../..
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name YCM
+    --repo-url https://github.com/robotology/ycm
+    --repo-checkout "${ROBOTOLOGY_CHECKOUT:-${YCM_CHECKOUT:-master}}"
 
   #-- Install YARP
-  - git clone --branch="$ROBOTOLOGY_CHECKOUT" https://github.com/robotology/yarp
-  - cd yarp && mkdir build && cd build
-  - cmake .. -DSKIP_ACE=ON -DCREATE_LIB_MATH=ON -DCREATE_DEVICE_LIBRARY_MODULES=ON -DENABLE_yarpmod_OpenNI2DeviceServer=ON -DOpenNI2_INCLUDE_DIR=/usr/local/include/OpenNI2/ -DOpenNI2_LIBRARY=/usr/local/lib/libOpenNI2.so -DNITE2_INCLUDE_LOCAL=/usr/local/include/NiTE-Linux-x64-2.2 -DNITE2_LIBRARY=/usr/local/lib/libNiTE2.so
-  - make -j$(nproc)
-  - sudo make install
-  - cd ../..
-
-  # see http://gronlier.fr/blog/2015/01/adding-code-coverage-to-your-c-project/
-  - if [ "$CXX" = "g++" ]; then wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.11.orig.tar.gz; fi
-  - if [ "$CXX" = "g++" ]; then tar xf lcov_1.11.orig.tar.gz; fi
-  - if [ "$CXX" = "g++" ]; then sudo make -C lcov-1.11/ install; fi
-  - if [ "$CXX" = "g++" ]; then gem install coveralls-lcov; fi
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name YARP
+    --repo-url https://github.com/robotology/yarp
+    --repo-checkout "${ROBOTOLOGY_CHECKOUT:-$YARP_CHECKOUT}"
+    --additional-cmake-options "-DSKIP_ACE:BOOL=ON -DCREATE_LIB_MATH:BOOL=ON -DCREATE_DEVICE_LIBRARY_MODULES:BOOL=ON -DENABLE_yarpmod_OpenNI2DeviceServer:BOOL=ON -DOpenNI2_INCLUDE_DIR:PATH=/usr/local/include/OpenNI2 -DOpenNI2_LIBRARY:FILEPATH=/usr/local/lib/libOpenNI2.so -DNITE2_INCLUDE_LOCAL:PATH=/usr/local/include/NiTE-Linux-x64-2.2 -DNITE2_LIBRARY:FILEPATH=/usr/local/lib/libNiTE2.so"
+    --prepend-to-linker-path lib
+    --additional-export-paths "YARP_DATA_DIRS;share/yarp"
+
+  #-- Install color-debug
+  - >
+    source "$TRAVIS_BUILD_DIR/scripts/travis/git-cache-dependency.sh"
+    --package-name COLOR_DEBUG
+    --repo-url https://github.com/roboticslab-uc3m/color-debug
+    --repo-checkout master
 
 before_script:
-  - mkdir -p build && cd build
-  - cmake .. -DENABLE_coverage=ON
+  - mkdir -p "$TRAVIS_BUILD_DIR/build" && cd "$_"
+  - cmake .. -DCMAKE_INSTALL_PREFIX="$PWD/install" -DENABLE_coverage:BOOL=ON
+  - make install
+  - export LD_LIBRARY_PATH=$PWD/install/lib:$LD_LIBRARY_PATH
+  - export YARP_DATA_DIRS=$PWD/install/share/roboticslab-vision:$YARP_DATA_DIRS
 
 script:
-  - make && sudo make install && sudo ldconfig
+  - echo "No unit tests available"
 
 after_success:
-  #  - cd ${TRAVIS_BUILD_DIR}
-  # capture coverage info
-  - if [ "$CXX" = "g++" ]; then lcov --directory . --capture --output-file coverage.info; fi
-  # filter out system and test code
-  - if [ "$CXX" = "g++" ]; then lcov --remove coverage.info  '/usr/*' 'test/*' --output-file coverage.info; fi
-  # debug before upload
-  - if [ "$CXX" = "g++" ]; then lcov --list coverage.info; fi
-  # uploads to coveralls
-  #- if [ "$CXX" = "g++" ]; then coveralls-lcov --source-encoding=ISO-8859-1 --repo-token InWqFy6VWMrEUzr2Rn8XV0cGqv3wXWo2d coverage.info; fi
-  - if [ "$CXX" = "g++" ]; then coveralls-lcov --source-encoding=ISO-8859-1 coverage.info; fi
-
-  - sudo make uninstall
+  #-- Code coverage
+  - if [ "$CXX" = "g++" ]; then source "$TRAVIS_BUILD_DIR/scripts/travis/coverage.sh"; fi
+
+  - make uninstall

scripts/travis/coverage.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set +e
+
+# see http://gronlier.fr/blog/2015/01/adding-code-coverage-to-your-c-project/
+
+#-- Capture coverage info
+lcov --directory . --capture --output-file coverage.info
+
+#-- Filter out system and test code
+lcov --remove coverage.info  '/usr/*' 'tests/*' --output-file coverage.info
+
+#-- Debug before upload
+lcov --list coverage.info
+
+#-- Upload to coveralls
+coveralls-lcov --source-encoding=ISO-8859-1 coverage.info

scripts/travis/git-cache-dependency.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+set -e
+
+#-- Initialize variables
+package_name=
+repo_url=
+repo_checkout=
+clone_only_branches="master devel develop production"
+cmake_home_dir=
+additional_cmake_options=
+prepend_to_linker_path=
+prepend_to_standard_path=
+additional_export_paths=
+
+#-- Available getopt long option names
+#-- https://gist.github.com/magnetikonline/22c1eb412daa350eeceee76c97519da8
+ARGUMENT_LIST=(
+    "package-name"
+    "repo-url"
+    "repo-checkout"
+    "clone-only-branches"
+    "cmake-home-dir"
+    "additional-cmake-options"
+    "prepend-to-linker-path"
+    "prepend-to-standard-path"
+    "additional-export-paths"
+)
+
+#-- Read arguments
+opts=$(getopt \
+    --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")" \
+    --name "$(basename "${BASH_SOURCE[0]}")" \
+    --options "" \
+    -- "$@"
+)
+
+eval set -- $opts
+
+#-- Parse options
+while [ "$#" -gt 1 ]; do
+    in=$(echo "$1" | sed -e 's/^--//')
+    for v in "${ARGUMENT_LIST[@]}"; do
+        if [ "$v" = "$in" ]; then
+            declare "$(echo "$v" | tr '-' '_')"="$2"
+            break
+        fi
+    done
+    shift 2
+done
+
+#-- Check required arguments
+if [ -z "$package_name" ] || [ -z "$repo_url" ] || [ -z "$repo_checkout" ]; then
+    echo "Missing required options. Traceback:"
+    for v in "${ARGUMENT_LIST[@]}"; do
+        v_param=$(echo $v | tr '-' '_')
+        echo "  --$v=${!v_param}"
+    done
+    return 1
+fi
+
+#-- Configure paths
+repo_source_dir=~/"$package_name-$repo_checkout"
+repo_build_dir="$repo_source_dir/build"
+repo_cache_dir="$CACHE_DIR/$package_name-$repo_checkout"
+cmake_home_dir="$repo_source_dir/$cmake_home_dir"
+
+#-- Configure CMake command line options
+repo_cmake_options="$additional_cmake_options -DCMAKE_INSTALL_PREFIX:PATH=$repo_cache_dir"
+
+is_clone_only_branch=false
+
+for branch in $clone_only_branches; do
+    if [ "$repo_checkout" = "$branch" ]; then
+        is_clone_only_branch=true
+        break
+    fi
+done
+
+if $is_clone_only_branch; then
+
+    #-- Clone, build and store in cache
+
+    echo "Cloning $package_name's $repo_checkout branch"
+    git clone --depth=1 --branch="$repo_checkout" "$repo_url" "$repo_source_dir"
+    last_commit_sha=$(git -C "$repo_source_dir" rev-parse HEAD)
+
+    if [ ! -d "$repo_cache_dir" ] || \
+       [ ! -f "$repo_cache_dir/.last_commit_sha" ] || \
+       [ ! "$(cat $repo_cache_dir/.last_commit_sha)" = "$last_commit_sha" ];
+    then
+        echo "$package_name not in cache or not the latest commit of $repo_checkout branch"
+        rm -rf "$repo_cache_dir"/*
+        cmake -H"$cmake_home_dir" -B"$repo_build_dir" $repo_cmake_options
+        make -C "$repo_build_dir" -j$(nproc) install
+        echo "$last_commit_sha" > "$repo_cache_dir/.last_commit_sha"
+    else
+        echo "$package_name found in cache ($(cat $repo_cache_dir/.last_commit_sha))"
+    fi
+
+else
+
+    #-- Download zipped file from archive, build and store in cache
+
+    if [ ! -d "$repo_cache_dir" ]; then
+        echo "Downloading $package_name $repo_checkout from archive"
+        wget -q "$repo_url/archive/$repo_checkout.tar.gz" -P "$repo_source_dir"
+        tar xzf "$repo_source_dir/$repo_checkout.tar.gz" -C "$repo_source_dir" --strip-components=1
+        cmake -H"$cmake_home_dir" -B"$repo_build_dir" $repo_cmake_options
+        make -C "$repo_build_dir" -j$(nproc) install
+    else
+        echo "$package_name $repo_checkout already in cache"
+    fi
+
+fi
+
+#-- Make installed package discoverable by CMake's find_package() command
+export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH:$repo_cache_dir"
+
+#-- Miscellanea, prepends stuff to LD_LIBRARY_PATH and PATH
+
+if [ ! -z "$prepend_to_linker_path" ]; then
+    export LD_LIBRARY_PATH="$repo_cache_dir/$prepend_to_linker_path:$LD_LIBRARY_PATH"
+fi
+
+if [ ! -z "$prepend_to_standard_path" ]; then
+    export PATH="$repo_cache_dir/$prepend_to_standard_path:$PATH"
+fi
+
+#-- Expands lists of values of the form 'VAR1;val1;VAR2;val2;...' to:
+#--  export VAR1=val1
+#--  export VAR2=val2
+#--  ...
+if [ ! -z "$additional_export_paths" ]; then
+    IFS=';' read -ra ITEMS <<< "$additional_export_paths"
+    for (( count=0; count<"${#ITEMS[@]}"; count+=2 )); do
+        export "${ITEMS[$count]}"="$repo_cache_dir/${ITEMS[$(( $count+1 ))]}:${!ITEMS[$count]}"
+    done
+fi