hadoop集群搭建（hdfs）

本文轉載自查看原文 2018-03-14 13:43 3946 hdfs

　　（搭建hadoop集群的前提是服務器已成功安裝jdk以及服務器之間已設置免密碼登錄，服務器之間的免密碼登錄可參考《linux服務器間ssh免密碼登錄》）

1、下載hadoop安裝包

　　wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-3.0.0/hadoop-3.0.0-src.tar.gz

2、解壓安裝包

　　tar zxvf hadoop-3.0.0-src.tar.gz

3、配置hadoop的環境變量

　　vi /etc/profile（三台機器）

　　增加以下配置

#Hadoop 3.0
export HADOOP_PREFIX=/home/hadoop/hadoop-3.0.0
export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin

export HADOOP_COMMON_HOME=$HADOOP_PREFIX 
export HADOOP_HDFS_HOME=$HADOOP_PREFIX 
export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
export HADOOP_YARN_HOME=$HADOOP_PREFIX 

export HADOOP_INSTALL=$HADOOP_PREFIX
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_PREFIX/lib/native 
export HADOOP_CONF_DIR=$HADOOP_PREFIX 
export HADOOP_PREFIX=$HADOOP_PREFIX 
export HADOOP_LIBEXEC_DIR=$HADOOP_PREFIX/libexec 
export JAVA_LIBRARY_PATH=$HADOOP_PREFIX/lib/native:$JAVA_LIBRARY_PATH 
export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop

　　　　source /etc/profile

4、修改配置文件

　　vi /etc/hosts(三台機器)

　　　　增加以下配置

10.9.1.101   node101
10.9.1.102   node102
10.9.1.103   node103

　　vi /home/hadoop/hadoop-3.0.0/etc/hadoop/core-site.xml（三台機器）

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
  <property>
     <name>fs.defaultFS</name>
     <value>hdfs://node101:9000</value> 
     <description>HDFS的URI，文件系統://namenode標識:端口</description>
  </property>
  <property>
     <name>hadoop.tmp.dir</name>
     <value>/opt/hadoop</value>
     <description>namenode上傳到hadoop的臨時文件夾</description>
  </property>
  <property>
       <name>fs.checkpoint.period</name>
       <value>3600</value>
       <description>用來設置檢查點備份日志的最長時間</description>
  </property>
</configuration>

　　vi /home/hadoop/hadoop-3.0.0/etc/hadoop/hdfs-site.xml（三台機器）

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
 <property>
   <name>dfs.replication</name>
   <value>3</value> 
   <description>副本個數，默認配置是3，應小於datanode機器數量</description>
 </property>
 <property>
    <name>dfs.name.dir</name>
    <value>/home/hadoop/hadoop-3.0.0/hdfs/name</value>
    <description>datanode上存儲hdfs名字空間元數據</description>
  </property>
 <property>
    <name>dfs.data.dir</name>
    <value>/home/hadoop/hadoop-3.0.0/hdfs/data</value>
    <description>datanode上數據塊的物理存儲位置</description>
 </property>
</configuration>