rlsd2-autobuild

#!/bin/sh
#description: build all rlsd2 packages and save them to ./rlsd2/$(uname -m)/pkg

bdir="$(cd "$(dirname "${0}")" && pwd)"
alert_mails="m#%#javier.io"
PS4="> "

#crontab jobs run in a very sparse environment
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin:/sbin"

_basename() {
    [ -z "${1}" ] && return 1 || _basename__name="${1}"
    [ -z "${2}" ] || _basename__suffix="${2}"
    case "${_basename__name}" in
        /*|*/*) _basename__name="$(expr "${_basename__name}" : '.*/\([^/]*\)')" ;;
    esac

    if [ -n "${_basename__suffix}" ] && [ "${#_basename__name}" -gt "${#2}" ]; then
        if [ X"$(printf "%s" "${_basename__name}" | cut -c"$((${#_basename__name} - ${#_basename__suffix} + 1))"-"${#_basename__name}")" \
           = X"$(printf "%s" "${_basename__suffix}")" ]; then
            _basename__name="$(printf "%s" "${_basename__name}" | cut -c1-"$((${#_basename__name} - ${#_basename__suffix}))")"
        fi
    fi

    printf "%s\\n" "${_basename__name}"
}

_strreplace() {
    #replace param in string, return 1 on failure
    #http://www.unix.com/shell-programming-and-scripting/124160-replace-word-string.html
    _strreplace__orig="${1}"
    [ -n "${3}" ] || return 1 #nothing to search for: error
    _strreplace__srch="${2}"  #pattern to replace
    _strreplace__rep="${3}"   #replacement string
    case "${_strreplace__orig}" in
        *"${_strreplace__srch}"*) #if pattern exists in the string
            _strreplace__sr1="${_strreplace__orig%%$_strreplace__srch*}" #take the string preceding the first match
            _strreplace__sr2="${_strreplace__orig#*$_strreplace__srch}"  #and the string following the first match
            _strreplace__orig="${_strreplace__sr1}${_strreplace__rep}${_strreplace__sr2}" #and sandwich the replacement string between them
            ;;
    esac
    printf "%s" "${_strreplace__orig}"
}

_warning() {
    [ -z "${1}" ] && return 1
    printf "%b\\n" "${*}" >&2
    if command -v "mailgun" >/dev/null; then
        for mail in ${alert_mails}; do
            mail="$(printf "%s\\n" "${mail}" | sed "s:#%#:@:")"
            mailgun --subject "${progname}: WARNING - ${1}" "${mail}" "$(
            printf "+---Trace-----------------------------\\n"
            printf "%b\\n" "${*}"
            printf "+-------------------------------------\\n"
            )" >/dev/null
        done
    fi
}

_die() {
    [ -z "${1}" ] && return 1
    printf "%b\\n" "${*}" >&2
    if command -v "mailgun" >/dev/null; then
        for mail in ${alert_mails}; do
            mail="$(printf "%s\\n" "${mail}" | sed "s:#%#:@:")"
            mailgun --subject "${progname}: FAILED" "${mail}" "$(
            printf "+---Trace-----------------------------\\n"
            printf "%b\\n" "${*}"
            printf "+-------------------------------------\\n"
            )" >/dev/null
        done
    fi
    exit 1
}

_clone() {
    [ -z "${1}" ] && return 1
    [ -z "${2}" ] && _clone__untarget="${bdir}/$(_basename "${1}")" || _clone__untarget="${2}"
    _clone__stdout="$(tempfile)"

    printf "%s\\n" "cloning: ${1}"
    if [ -d "${_clone__untarget}" ]; then
         (cd "${_clone__untarget}" && git pull) > "${_clone__stdout}" 2>&1
    else
        git clone --depth=1 "${1}" "${_clone__untarget}" > "${_clone__stdout}" 2>&1
    fi
    if [ ! -d "${_clone__untarget}" ]; then
        _clone__stdoutb="$(cat "${_clone__stdout}")"; rm -rf "${_clone__stdout}"
        _die "failed to clone ${1} to ${_clone__untarget}\\n" "$(printf "%s\\n" "${_clone__stdoutb}")"
    fi
    rm -f "${_clone__stdout}"
}

_wget() {
    [ -z "${1}" ] && return 1
    [ -z "${2}" ] && _wget__untarget="${bdir}/$(_basename "${1}")" || _wget__untarget="${2}"
    _wget__stdout="$(tempfile)"

    printf "%s\\n" "fetching: ${1}"
    if [ ! -f "${_wget__untarget}" ]; then
        wget -q "${1}" -O "${_wget__untarget}" > "${_wget__stdout}" 2>&1
    fi
    if [ ! -f "${_wget__untarget}" ]; then
        _wget__stdoutb="$(cat "${_wget__stdout}")"; rm -rf "${_wget__stdout}"
        _die "failed to fetch ${1} to ${_wget__untarget}\\n" "$(printf "%s\\n" "${_wget__stdoutb}")"
    fi
    rm -f "${_wget__stdout}"
}

_untar() {
    [ -z "${1}" ] && return 1
    [ -z "${2}" ] && _untar__target="${bdir}/$(_basename "${1}")" || _untar__target="${2}"
    _untar__stdout="$(tempfile)"

    if [ ! -d "${_untar__target}" ]; then
        (mkdir -p "${_untar__target}" && tar -xf "${1}" -C "${_untar__target}") > "${_untar__stdout}" 2>&1
    fi
    if [ ! -d "${_untar__target}" ]; then
        _untar__stdoutb="$(cat "${_untar__stdout}")"; rm -rf "${_untar__stdout}"
        _die "failed to untar ${1} to ${_untar__target}\\n" "$(printf "%s\\n" "${_untar__stdoutb}")"
    fi
    rm -f "${_untar__stdout}"
}

_cp() {
    [ -z "${1}" ] && return 1
    [ -z "${2}" ] && _cp_target="${bdir}/$(_basename "${1}")" || _cp_target="${2}"
    _cp__stdout="$(tempfile)"

    if [ ! -d "${_cp_target}" ]; then
        cp -rp "${1}" "${_cp_target}" > "${_cp__stdout}" 2>&1
    fi
    if [ ! -d "${_cp_target}" ] && [ ! -f "${_cp_target}" ]; then
        _cp__stdoutb="$(cat "${_cp__stdout}")"; rm -rf "${_cp__stdout}"
        _die "failed to cp ${1} to ${_cp_target}\\n" "$(printf "%s\\n" "${_cp__stdoutb}")"
    fi
    rm -f "${_cp__stdout}"
}

_install_ubuntu_pkg() {
    cd "${bdir}/rlsd2-build/"
    _ibpkg__stdout="$(tempfile)"
    for _install_ubuntu_pkg__pkg; do
        printf "%s\\n" "installing ubuntu pkg: ${_install_ubuntu_pkg__pkg}"
        ./chroot-i686-install.sh "${_install_ubuntu_pkg__pkg}" > "${_ibpkg__stdout}" 2>&1 || {
        _ibpkg__stdoutb="$(cat "${_ibpkg__stdout}")"; rm -rf "${_ibpkg__stdout}"
        _die "failed to install ${_install_ubuntu_pkg__pkg} in the init environment\\n" "$(printf "%s\\n" "${_ibpkg__stdoutb}")"
        }
    done
    rm -rf "${_ibpkg__stdout}"
}

_repackage() {
    [ -z "${2}" ] && return 1
    _repackage__bname="$(_basename "${1}" .tar.gz)"
    _repackage__dname="/tmp/"${progname}".repackage"
    _repackage_init() {
        mkdir -p "${_repackage__dname}"
        [ -d "${_repackage__dname}/${_repackage__bname}" ] && return 0
        (cd  "${_repackage__dname}" && mkdir "${_repackage__bname}" && tar xf "${1}" -C "${_repackage__bname}")
    }
    case "${2}" in
        bz2) [ -f "${_repackage__dname}/${_repackage__bname}.tar.bz2" ] && return 0
            _repackage_init "${1}"
            (cd  "${_repackage__dname}" && tar -C "${_repackage__bname}" -jcf "${_repackage__bname}".tar.bz2 .)
            cp "${_repackage__dname}/${_repackage__bname}.tar.bz2" .
            ;;
         xz) [ -f "${_repackage__dname}/${_repackage__bname}.tar.xz" ]  && return 0
            _repackage_init "${1}"
            (cd  "${_repackage__dname}" && tar -C "${_repackage__bname}" -Jcf "${_repackage__bname}".tar.xz .)
            cp "${_repackage__dname}/${_repackage__bname}.tar.xz"  .
            ;;
          *) rm -rf "${_repackage__dname}" ;;
    esac
}

progname="$(_basename ${0})"
nonroot_user="$(getent passwd | awk -F: '{if ($3 >= 1000 && $3 < 60000) print $1}'|head -1)"

if ! command -v "git" >/dev/null; then
    _die "you need to install 'git' to run this program"
fi

[ X"$(whoami)" != X"root" ] && _die "this program can only be run by root (because of chroot), exiting."

exec 9>/tmp/"${progname}".lock #verify that only one instance is running
if ! flock -n 9; then          #http://mywiki.wooledge.org/BashFAQ/045
    _die "${progname}: another instance is running";
fi

#get sources
_clone https://github.com/dimkr/rlsd2           "${bdir}/rlsd2-build/upstream-rlsd2"
_clone https://github.com/minos-org/rlsd2-delta "${bdir}/rlsd2-build/minos-rlsd2"
_wget  "https://googledrive.com/host/0ByFmYLqbl00ofkxoTXNXMXZKNkJiZXdaUlNCY05XanVqMVpUNEVOeUIxODhGUVpmd281dEE/precise-i386-base.tar.gz" \
       "${bdir}/rlsd2-build/precise-i386-base.tar.gz"
#_wget  "https://googledrive.com/host/0ByFmYLqbl00ofkxoTXNXMXZKNkJiZXdaUlNCY05XanVqMVpUNEVOeUIxODhGUVpmd281dEE/precise-amd64-base.tar.gz" \
       #"${bdir}/rlsd2-build/precise-amd64-base.tar.gz"

#setup base environment
_untar "${bdir}"/rlsd2-build/precise-i386-base.tar.gz "${bdir}/rlsd2-build/native-i386/"
#_untar "${bdir}"/rlsd2-build/precise-amd64-base.tar.gz "${bdir}/rlsd2-build/native-x86_64/"

#install base pkgs: https://github.com/dimkr/rlsd2/blob/master/packer/www/preseed.cfg
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-i686.sh"         "${bdir}/rlsd2-build/chroot-i686.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-i686-install.sh" "${bdir}/rlsd2-build/chroot-i686-install.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-i686-build.sh"   "${bdir}/rlsd2-build/chroot-i686-build.sh"
#_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-x86_64.sh"         "${bdir}/rlsd2-build/chroot-x86_64.sh"
#_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-x86_64-install.sh" "${bdir}/rlsd2-build/chroot-x86_64-install.sh"
#_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/chroot-x86_64-build.sh"   "${bdir}/rlsd2-build/chroot-x86_64-build.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/env-setup.sh"   "${bdir}/rlsd2-build/native-i386/bin/env-setup.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/env-build.sh"   "${bdir}/rlsd2-build/native-i386/bin/env-build.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/env-install.sh" "${bdir}/rlsd2-build/native-i386/bin/env-install.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/env-hooks.sh"   "${bdir}/rlsd2-build/native-i386/bin/env-hooks.sh"
_cp "${bdir}/rlsd2-build/minos-rlsd2/scripts/rlsd2-pkg"      "${bdir}/rlsd2-build/native-i386/bin/rlsd2-pkg"

_install_ubuntu_pkg $(wget --no-check-certificate -qO- \
    https://raw.githubusercontent.com/dimkr/rlsd2/master/packer/www/preseed.cfg | \
    awk '/pkgsel.?include/ {$1=""; $2=""; $3=""; print $0}' | \
    awk '{ for(i=1;i<=NF;i++) if ($i == "extlinux" || $i == "qemu-utils")  $i=""; else printf "%s ", $i }')

#setup rlsd2
mkdir -p "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2"
mkdir -p "${bdir}/rlsd2-build/native-i386/var/spool/rlsd2-pkgs"
rsync -aq --update "${bdir}/rlsd2-build/upstream-rlsd2/" "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2/" || \
    _die  "rlsd2 setup failed (rsync upstream-rlsd2)"
#remove previous patched files
rsync -aqI --delete "${bdir}/rlsd2-build/upstream-rlsd2/rules/" "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2/rules/" || \
    _die  "rlsd2 setup failed (rsync rules)"
printf "%s\\n" "check_certificate = off" > "${bdir}/rlsd2-build/native-i386/etc/wgetrc"

#apply minos-rlsd2 delta
rsync -aq "${bdir}/rlsd2-build/minos-rlsd2/" "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2/" || \
    _die  "rlsd2 delta setup failed"

#apply patches
for delta_patch in "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2/patches/*.patch; do
    [ -f "${delta_patch}" ] || continue
    _patch_outout="$(tempfile)"
    basename_delta_patch="$(_basename "${delta_patch}" .patch)"
    printf "%s\\n" "applying patch to: ${basename_delta_patch}"
    (cd "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2/rules/"${basename_delta_patch}" &&
    patch < "${delta_patch}") > "${_patch_outout}" 2>&1 ||
    _warning "${basename_delta_patch} patch failed" "\\n$(tail -100 "${_patch_outout}")"
    rm -rf "${_patch_outout}"
done

#init, https://github.com/dimkr/rlsd2/blob/master/scripts/build_all
(cd "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2/" && ./scripts/gen_deps > Makefile.deps)

#build packages
for recipe in "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2/rules/*; do
    [ ! -d "${recipe}" ] && continue
    brecipe="$(_basename "${recipe}")"
    case "${brecipe}" in #skip these
        open-ath9k-htc-firmware|initramfs) continue ;;
    esac
    #only build updated recipes
    if ! diff -rq -x '.*' "${recipe}" "${bdir}/rlsd2-build/native-i386/var/lib/rlsd2.rules.reference/${brecipe}" >/dev/null 2>&1 || \
        [ ! -f "${recipe}/../../packages/${brecipe}".tar.gz ]; then
        rm -rf "${recipe}/../../packages/${brecipe}".tar.gz "${recipe}/../../sources/${brecipe}/*"
        printf "%s\\n" "building ${brecipe} ..."; _recipe__stdout="$(tempfile)"
        (cd "${bdir}/rlsd2-build/" && ./chroot-i686-build.sh "${brecipe}" > "${_recipe__stdout}" 2>&1)
        if [ -f "${recipe}"/../../packages/"${brecipe}".tar.gz ]; then
            cp "${recipe}"/../../packages/"${brecipe}".tar.gz "${bdir}"/rlsd2-build/native-i386/var/spool/rlsd2-pkgs/
            [ -d "${bdir}/rlsd2-build/minos-rlsd2/broken/${brecipe}" ] &&
            _warning "${brecipe} seems no longer broken" "\\n$(tail -100 "${_recipe__stdout}")"
        else
            [ -d "${bdir}/rlsd2-build/minos-rlsd2/broken/${brecipe}" ] ||
            _warning "${brecipe} failed to build" "\\n$(tail -100 "${recipe}/../../logs/${brecipe}.log")"
        fi
        rm -rf "${_recipe__stdout}"
    fi
done
umount "${bdir}"/rlsd2-build/native-i386/proc "${bdir}"/rlsd2-build/native-i386/sys 2>/dev/null

#create reference
rm -rf "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2.rules.reference
_cp "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2/rules "${bdir}"/rlsd2-build/native-i386/var/lib/rlsd2.rules.reference

#repack them in bz2 and xz archives
printf "%s\\n" "creating archives ..."
mkdir -p "${bdir}"/rlsd2-build/repackage-i686
for archive in "${bdir}"/rlsd2-build/native-i386/var/spool/rlsd2-pkgs/*.tar.gz; do
    [ ! -f "${archive}" ] && continue
    barchive="$(_basename "${archive}" .tar.gz)"; barchive="$(_strreplace "${barchive}" "#" "-")"
    _cp "${archive}" "${bdir}"/rlsd2-build/repackage-i686/"${barchive}".tar.gz
    #process only updated packages
    hash_orig="$(shasum -a 512 "${bdir}/rlsd2/i686/${barchive}".tar.gz 2>/dev/null|awk '{print $1}')"
    hash_new="$( shasum -a 512 "${bdir}/rlsd2-build/repackage-i686/${barchive}".tar.gz 2>/dev/null|awk '{print $1}')"
    [ X"${hash_new}" = X"${hash_orig}" ] && continue
    (
    cd "${bdir}"/rlsd2-build/repackage-i686
    _repackage "${bdir}"/rlsd2-build/repackage-i686/"${barchive}".tar.gz bz2
    _repackage "${bdir}"/rlsd2-build/repackage-i686/"${barchive}".tar.gz xz
    _repackage "${bdir}"/rlsd2-build/repackage-i686/"${barchive}".tar.gz cleanup
    )
done

#move packages
printf "%s\\n" "moving archives ...";
[ ! -d "${bdir}"/rlsd2/i686 ] && mkdir -p "${bdir}"/rlsd2/i686
[ ! -f "${bdir}"/rlsd2/lock ] && touch "${bdir}"/rlsd2/lock

for archive in "${bdir}"/rlsd2-build/repackage-i686/*.tar.gz  \
               "${bdir}"/rlsd2-build/repackage-i686/*.tar.bz2 \
               "${bdir}"/rlsd2-build/repackage-i686/*.tar.xz; do
    [ ! -f "${archive}" ] && continue
    mv "${archive}" "${bdir}"/rlsd2/i686/
done

#create index
printf "%s\\n" "creating index ...";
(cd "${bdir}"/rlsd2/i686/ && rm -rf sha512sum.* md5sum.*)
(cd "${bdir}"/rlsd2/i686/ && shasum -a 512 *tar* > sha512sum.txt 2>/dev/null)
(cd "${bdir}"/rlsd2/i686/ && md5sum *tar* > md5sum.txt 2>/dev/null)
[ X"$(ls "${bdir}"/rlsd2/i686/*tar*|wc -l)" = X"$(wc -l "${bdir}"/rlsd2/i686/sha512sum.txt|awk '{print $1}')" ] || \
    _die "Package entries don't match total archives\\n"     \
    "sha512 entries:$(wc -l "${bdir}"/rlsd2/i686/sha512sum.txt)\\n" \
    "archives      :$(ls "${bdir}"/rlsd2/i686/*tar*|wc -l)"

(cd "${bdir}"/rlsd2/i686/ && tar -zcf sha512sum.tar.gz  sha512sum.txt)
(cd "${bdir}"/rlsd2/i686/ && tar -jcf sha512sum.tar.bz2 sha512sum.txt)
(cd "${bdir}"/rlsd2/i686/ && tar -Jcf sha512sum.tar.xz  sha512sum.txt)
(cd "${bdir}"/rlsd2/i686/ && tar -zcf md5sum.tar.gz  md5sum.txt)
(cd "${bdir}"/rlsd2/i686/ && tar -jcf md5sum.tar.bz2 md5sum.txt)
(cd "${bdir}"/rlsd2/i686/ && tar -Jcf md5sum.tar.xz  md5sum.txt)

#use x86 binaries for x64, x64 binaries could be produced in the future when more hardware resources are available
[ -d "${bdir}"/rlsd2/x86_64 ] && rm -rf "${bdir}"/rlsd2/x86_64/* || mkdir -p "${bdir}"/rlsd2/x86_64
ln -s "${bdir}"/rlsd2/i686/* "${bdir}"/rlsd2/x86_64/

chown -R "${nonroot_user}:${nonroot_user}" "${bdir}"/rlsd2/
rm "${bdir}"/rlsd2/lock