搭建Hadoop Yarn环境 (ARM)

[Yikun]

---

date: 2020-08-15

ARM上搭建Hadoop Yarn的指导。

1. 安装依赖

yum update -y
yum install -y \
    autoconf \
    automake \
    libtool \
    cmake \
    pkg-config \
    curl \
    sudo \
    git \
    hostname \
    bind-utils \
    openssh-server \
    snappy \
    snappy-devel \
    bzip2 \
    libzip-devel \
    java-1.8.0-openjdk \
    java-1.8.0-openjdk-devel

2. 新增Hadoop用户

groupadd hadoop
useradd -m -d /home/hadoop -s /bin/bash hadoop -g hadoop
echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

3. 下载

curl -L https://downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz | tar zx

4. 环境变量准备

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.265.b01-6.oe1.aarch64
export HADOOP_HOME=/home/hadoop/hadoop-3.3.0

5. 配置添加
   cd $HADOOP_HOME/etc/hadoop/
   core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.name.dir</name>
        <value>/home/hadoop/dfs/name</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/home/hadoop/dfs/data</value>
    </property>
</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!--Deploy Hadoop on K8s, we need to specify a certain port range and expose by k8s Service-->
    <property>
        <name>yarn.app.mapreduce.am.job.client.port-range</name>
        <value>60000-60010</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.webapp.port-range</name>
        <value>60011-60020</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>
    <!--
    <property>
      <name>mapreduce.map.output.compress</name>
      <value>true</value>
    </property>
    <property>
      <name>mapreduce.map.output.compress.codec</name>
      <value>org.apache.hadoop.io.compress.ZStandardCodec</value>
    </property>
    -->
</configuration>

yarn-site.xml

<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>localhost</value>
    </property>
    <property>
        <name>yarn.nodemanager.address</name>
        <value>0.0.0.0:40555</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
</configuration>

6. 启动服务
   先格式化hdfs

hdfs namenode -format

启动所有服务

start-all.sh