magpie-setup-core

#!/bin/bash
#############################################################################
#  Copyright (C) 2013-2015 Lawrence Livermore National Security, LLC.
#  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
#  Written by Albert Chu <chu11@llnl.gov>
#  LLNL-CODE-644248
#  
#  This file is part of Magpie, scripts for running Hadoop on
#  traditional HPC systems.  For details, see https://github.com/llnl/magpie.
#  
#  Magpie is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#  
#  Magpie is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with Magpie.  If not, see <http://www.gnu.org/licenses/>.
#############################################################################

# This script sets up core configuration files for jobs.  For the most
# part, it shouldn't be editted.  See job submission files for
# configuration details.

source ${MAGPIE_SCRIPTS_HOME}/magpie/exports/magpie-exports-submission-type
source ${MAGPIE_SCRIPTS_HOME}/magpie/exports/magpie-exports-dirs
source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-hadoop-helper

#
# Setup a whole bunch of environment variables and directories
# 

# Create conf & log dirs node specific
Magpie_make_all_local_dirs_node_specific

mkdir -p $MAGPIE_LOCAL_CONF_DIR
if [ $? -ne 0 ] ; then
    echo "mkdir failed making ${MAGPIE_LOCAL_CONF_DIR}"
    exit 1
fi

mkdir -p $MAGPIE_LOCAL_SCRATCHSPACE_DIR
if [ $? -ne 0 ] ; then
    echo "mkdir failed making ${MAGPIE_LOCAL_SCRATCHSPACE_DIR}"
    exit 1
fi

#
# Setup primary conf, log, and local scratchspace directories for projects
#

magpieprojects="HADOOP PIG MAHOUT HBASE PHOENIX SPARK KAFKA ZEPPELIN STORM TACHYON ZOOKEEPER"

for project in ${magpieprojects}
do
    setupvar="${project}_SETUP"
    confdirvar="${project}_CONF_DIR"
    logdirvar="${project}_LOG_DIR"
    piddirvar="${project}_PID_DIR"
    localscratchspacedirvar="${project}_LOCAL_SCRATCHSPACE_DIR"

    if [ "${!setupvar}" == "yes" ]
    then
        if [ "${!confdirvar}X" != "X" ]
        then
            mkdir -p ${!confdirvar}
            if [ $? -ne 0 ] ; then
                echo "mkdir failed making ${!confdirvar}"
                exit 1
            fi
        fi

        if [ "${!logdirvar}X" != "X" ]
        then
            mkdir -p ${!logdirvar}
            if [ $? -ne 0 ] ; then
                echo "mkdir failed making ${!logdirvar}"
                exit 1
            fi
        fi

        if [ "${!piddirvar}X" != "X" ]
        then
            mkdir -p ${!piddirvar}
            if [ $? -ne 0 ] ; then
                echo "mkdir failed making ${!piddirvar}"
                exit 1
            fi
        fi

        if [ "${!localscratchspacedirvar}X" != "X" ]
        then
            mkdir -p ${!localscratchspacedirvar}
            if [ $? -ne 0 ] ; then
                echo "mkdir failed making ${!localscratchspacedirvar}"
                exit 1
            fi
        fi
    fi
done

#
# Setup/Calculate Core Master/Slave files
#

hash hostlist 2>/dev/null
if [ $? -eq 0 ]
then
    hostrangescript="hostlist"
    hostrangescriptoptions="-e -d"
else
    if [ -x ${MAGPIE_SCRIPTS_HOME}/bin/magpie-expand-nodes ]
    then
        hostrangescript="${MAGPIE_SCRIPTS_HOME}/bin/magpie-expand-nodes"
        hostrangescriptoptions=""
    else
        echo "Cannot find tool to expand host ranges"
        exit 1
    fi
fi

${hostrangescript} ${hostrangescriptoptions} ${MAGPIE_NODELIST} | head -1 > ${MAGPIE_LOCAL_CONF_DIR}/tmp_masters
${hostrangescript} ${hostrangescriptoptions} ${MAGPIE_NODELIST} | tail -n+2 > ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves

if [ "${HADOOP_SETUP}" == "yes" ] \
    && Magpie_hadoop_filesystem_mode_is_hdfs_type \
    && Magpie_hdfs_federation_enabled
then
    extranamenodecount=`expr ${HDFS_FEDERATION_NAMENODE_COUNT} - 1`

    head -${extranamenodecount} ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves > ${MAGPIE_LOCAL_CONF_DIR}/namenode_hdfs_federation

    tail -n+${HDFS_FEDERATION_NAMENODE_COUNT} ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves > ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves_tmp

    cp ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves_tmp ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves
fi

if [ "${ZOOKEEPER_SETUP}" == "yes" ]
then
    tmpslavecount=`cat ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves | wc -l`
    jobslavecount=`expr ${tmpslavecount} - ${ZOOKEEPER_REPLICATION_COUNT}`
    zookeeperslavestart=`expr ${jobslavecount} + 1`

    cp ${MAGPIE_LOCAL_CONF_DIR}/tmp_masters ${MAGPIE_LOCAL_CONF_DIR}/job_masters

    if [ "${ZOOKEEPER_SHARE_NODES}" != "yes" ] 
    then
        head -${jobslavecount} ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves > ${MAGPIE_LOCAL_CONF_DIR}/job_slaves
    else
        cp ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves ${MAGPIE_LOCAL_CONF_DIR}/job_slaves
    fi

    tail -n+${zookeeperslavestart} ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves > ${ZOOKEEPER_CONF_DIR}/slaves 

    # The Zookeeper master is not an actual Zookeeper node, this is
    # predominantly to indicate to magpie that Zookeeper should be
    # launched via the "master" node
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${ZOOKEEPER_CONF_DIR}/masters
else
    cp ${MAGPIE_LOCAL_CONF_DIR}/tmp_masters ${MAGPIE_LOCAL_CONF_DIR}/job_masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/tmp_slaves ${MAGPIE_LOCAL_CONF_DIR}/job_slaves
fi

if [ "${HADOOP_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${HADOOP_CONF_DIR}/masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${HADOOP_CONF_DIR}/slaves
    if [ -f "${MAGPIE_LOCAL_CONF_DIR}/namenode_hdfs_federation" ]
    then
        cp ${MAGPIE_LOCAL_CONF_DIR}/namenode_hdfs_federation ${HADOOP_CONF_DIR}/namenode_hdfs_federation
    fi

    slavenodes=`cat ${HADOOP_CONF_DIR}/slaves`
    `touch ${HADOOP_CONF_DIR}/slaves_fqdn`
    dnsdomainname=`dnsdomainname`

    # We assume FQDN across all nodes is the same
    for node in ${slavenodes}
    do
        echo "${node}.${dnsdomainname}" >> ${HADOOP_CONF_DIR}/slaves_fqdn
    done
fi

if [ "${HBASE_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${HBASE_CONF_DIR}/masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${HBASE_CONF_DIR}/regionservers
fi

if [ "${PHOENIX_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${PHOENIX_CONF_DIR}/regionservers
fi

if [ "${SPARK_SETUP}" == "yes" ]
then
    # Create regardless of using yarn or not, needed for other code
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${SPARK_CONF_DIR}/masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${SPARK_CONF_DIR}/slaves
fi

if [ "${KAFKA_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${KAFKA_CONF_DIR}/masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${KAFKA_CONF_DIR}/slaves
fi

if [ "${ZEPPELIN_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${ZEPPELIN_CONF_DIR}/masters
fi

if [ "${STORM_SETUP}" == "yes" ]
then
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${STORM_CONF_DIR}/masters
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${STORM_CONF_DIR}/workers
fi

if [ "${TACHYON_SETUP}" == "yes" ]
then
    # Tachyon assumes setup node is master, so no need for masters
    # file but we still output it b/c there are functions in Magpie
    # that assume it exists 
    cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${TACHYON_CONF_DIR}/masters
    if [ "${TACHYON_WORKER_ON_MASTER}" == "yes" ]
    then
        cp ${MAGPIE_LOCAL_CONF_DIR}/job_masters ${TACHYON_CONF_DIR}/workers
        cat ${MAGPIE_LOCAL_CONF_DIR}/job_slaves >> ${TACHYON_CONF_DIR}/workers
    else
        cp ${MAGPIE_LOCAL_CONF_DIR}/job_slaves ${TACHYON_CONF_DIR}/workers
    fi
fi

exit 0