解压、改名 tar -zxvf hadoop-2.6.0.tar.gz mv hadoop-2.6.0 hadoop 验证hadoop安装成功 hadoop version
修改hadoop配置文件 [1] vi /usr/hadoop/hadoop/etc/hadoop/core-site.xml ############################################### <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <!-- 开启垃圾箱功能,1440分钟 --> <property> <name>fs.trash.interval</name> <value>1440</value> </property>
<!-- 指定hdfs的nameservice为ns1,是NameNode的URI。hdfs://主机名:端口/ --> <property> <name>fs.defaultFS</name> <value>hdfs://gagcluster:8020</value> </property>
<property> <name>io.file.buffer.size</name> <value>131072</value> </property>
<!-- 指定hadoop临时目录 --> <property> <name>hadoop.tmp.dir</name> <value>/usr/hadoop/storage/hadoop/tmp</value> </property>
<!--指定可以在任何IP访问--> <property> <name>hadoop.proxyuser.root.hosts</name> <value>*</value> </property>
<!--指定所有用户可以访问--> <property> <name>hadoop.proxyuser.root.groups</name> <value>*</value> </property>
<!-- 指定zookeeper地址 --> <property> <name>ha.zookeeper.quorum</name> <value>SA01:2181,SA02:2181,SA03:2181</value> </property> </configuration> #################################################
[2] vi /usr/hadoop/hadoop/etc/hadoop/hdfs-site.xml ################################################ <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <!--节点黑名单列表文件，用于下线hadoop节点 --> <property> <name>dfs.hosts.exclude</name> <value>/usr/hadoop/hadoop/etc/hadoop/exclude</value> </property>
<!--指定hdfs的block大小64M --> <property> <name>dfs.block.size</name> <value>67108864</value> </property>
<!--指定hdfs的nameservice为ns1，需要和core-site.xml中的保持一致 --> <property> <name>dfs.nameservices</name> <value>gagcluster</value> </property>
<!-- ns1下面有两个NameNode，分别是nn1，nn2 --> <property> <name>dfs.ha.namenodes.gagcluster</name> <value>nn1,nn2</value> </property>
<!-- nn1的RPC通信地址 --> <property> <name>dfs.namenode.rpc-address.gagcluster.nn1</name> <value>SA01:8020</value> </property>
<!-- nn1的http通信地址 --> <property> <name>dfs.namenode.http-address.gagcluster.nn1</name> <value>SA01:50070</value> </property>
<!-- nn2的RPC通信地址 --> <property> <name>dfs.namenode.rpc-address.gagcluster.nn2</name> <value>SA02:8020</value> </property>
<!-- nn2的http通信地址 --> <property> <name>dfs.namenode.http-address.gagcluster.nn2</name> <value>SA02:50070</value> </property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 --> <property> <name>dfs.namenode.shared.edits.dir</name> <value>qjournal://SA01:8485;SA02:8485;SA03:8485/gagcluster</value> </property>
<!-- 配置失败自动切换实现方式 --> <property> <name>dfs.client.failover.proxy.provider.gagcluster</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> </property>
<!-- 配置隔离机制 --> <property> <name>dfs.ha.fencing.methods</name> <value>sshfence</value> </property>
<!-- 使用隔离机制时需要ssh免密码登陆 --> <property> <name>dfs.ha.fencing.ssh.private-key-files</name> <value>/root/.ssh/id_rsa</value> </property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 --> <property> <name>dfs.journalnode.edits.dir</name> <value>/usr/hadoop/storage/hadoop/journal</value> </property>
<!--指定支持高可用自动切换机制--> <property> <name>dfs.ha.automatic-failover.enabled</name> <value>true</value> </property>
<!--指定namenode名称空间的存储地址--> <property> <name>dfs.namenode.name.dir</name> <value>/usr/hadoop/storage/hadoop/name</value> </property>
<!--指定datanode数据存储地址--> <property> <name>dfs.datanode.data.dir</name> <value>file:/usr/hadoop/storage/hadoop/data</value> </property>
<!--指定数据冗余份数--> <property> <name>dfs.replication</name> <value>2</value> </property>
<!--指定可以通过web访问hdfs目录--> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property>
<!--保证数据恢复 --> <property> <name>dfs.journalnode.http-address</name> <value>0.0.0.0:8480</value> </property>
<property> <name>dfs.journalnode.rpc-address</name> <value>0.0.0.0:8485</value> </property>
<property> <name>ha.zookeeper.quorum</name> <value>SA01:2181,SA02:2181,SA03:2181</value> </property> </configuration> ################################################# [3] cp /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml.template /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml vi /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml ################################################# <?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <!-- 配置MapReduce运行于yarn中 --> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property>
<!-- 配置 MapReduce JobHistory Server 地址 ，默认端口10020 --> <property> <name>mapreduce.jobhistory.address</name> <value>0.0.0.0:10020</value> </property>
<!-- 配置 MapReduce JobHistory Server web ui 地址， 默认端口19888 --> <property> <name>mapreduce.jobhistory.webapp.address</name> <value>0.0.0.0:19888</value> </property> </configuration> ###################################################
[4] vi /usr/hadoop/hadoop/etc/hadoop/yarn-site.xml ################################################### <?xml version="1.0"?> <configuration> <!--日志聚合功能--> <property> <name>yarn.log-aggregation-enable</name> <value>true</value> </property>
<!--在HDFS上聚合的日志最长保留多少秒。3天--> <property> <name>yarn.log-aggregation.retain-seconds</name> <value>259200</value> </property>
<!--rm失联后重新链接的时间--> <property> <name>yarn.resourcemanager.connect.retry-interval.ms</name> <value>2000</value> </property>
<!--开启resource manager HA,默认为false--> <property> <name>yarn.resourcemanager.ha.enabled</name> <value>true</value> </property>
<!--配置resource manager --> <property> <name>yarn.resourcemanager.ha.rm-ids</name> <value>rm1,rm2</value> </property>
<property> <name>ha.zookeeper.quorum</name> <value>SA01:2181,SA02:2181,SA03:2181</value> </property> <!--开启故障自动切换--> <property> <name>yarn.resourcemanager.ha.automatic-failover.enabled</name> <value>true</value> </property>
<property> <name>yarn.resourcemanager.hostname.rm1</name> <value>SA01</value> </property> <property> <name>yarn.resourcemanager.hostname.rm2</name> <value>SA02</value> </property>
<!--在namenode1上配置rm1,在namenode2上配置rm2,注意：一般都喜欢把配置好的文件远程复制到其它机器上，但这个在YARN的另一个机器上一定要修改--> <property> <name>yarn.resourcemanager.ha.id</name> <value>rm1</value> </property>
<!--开启自动恢复功能--> <property> <name>yarn.resourcemanager.recovery.enabled</name> <value>true</value> </property>
<!--配置与zookeeper的连接地址--> <property> <name>yarn.resourcemanager.zk-state-store.address</name> <value>SA01:2181,SA02:2181,SA03:2181</value> </property>
<property> <name>yarn.resourcemanager.store.class</name> <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> </property>
<property> <name>yarn.resourcemanager.zk-address</name> <value>SA01:2181,SA02:2181,SA03:2181</value> </property>
<property> <name>yarn.resourcemanager.cluster-id</name> <value>gagcluster-yarn</value> </property>
<!--schelduler失联等待连接时间--> <property> <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name> <value>5000</value> </property>
<!--配置rm1--> <property> <name>yarn.resourcemanager.address.rm1</name> <value>SA01:8132</value> </property>
<property> <name>yarn.resourcemanager.scheduler.address.rm1</name> <value>SA01:8130</value> </property>
<property> <name>yarn.resourcemanager.webapp.address.rm1</name> <value>SA01:8188</value> </property>
<property> <name>yarn.resourcemanager.resource-tracker.address.rm1</name> <value>SA01:8131</value> </property>
<property> <name>yarn.resourcemanager.admin.address.rm1</name> <value>SA01:8033</value> </property>
<property> <name>yarn.resourcemanager.ha.admin.address.rm1</name> <value>SA01:23142</value> </property>
<!--配置rm2--> <property> <name>yarn.resourcemanager.address.rm2</name> <value>SA02:8132</value> </property>
<property> <name>yarn.resourcemanager.scheduler.address.rm2</name> <value>SA02:8130</value> </property>
<property> <name>yarn.resourcemanager.webapp.address.rm2</name> <value>SA02:8188</value> </property>
<property> <name>yarn.resourcemanager.resource-tracker.address.rm2</name> <value>SA02:8131</value> </property>
<property> <name>yarn.resourcemanager.admin.address.rm2</name> <value>SA02:8033</value> </property>
<property> <name>yarn.resourcemanager.ha.admin.address.rm2</name> <value>SA02:23142</value> </property>
<property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property>
<property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property>
<property> <name>yarn.nodemanager.local-dirs</name> <value>/usr/hadoop/storage/yarn/local</value> </property>
<property> <name>yarn.nodemanager.log-dirs</name> <value>/usr/hadoop/storage/yarn/logs</value> </property>
<property> <name>mapreduce.shuffle.port</name> <value>23080</value> </property>
<!--故障处理类--> <property> <name>yarn.client.failover-proxy-provider</name> <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value> </property>
<property> <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name> <value>/yarn-leader-election</value> </property> </configuration>
#######################################################
配置DataNode节点 vi /usr/hadoop/hadoop/etc/hadoop/slaves ####################################################### slave141 slave142 slave143 Slave144 Slave145
vi /usr/hadoop/hadoop/etc/hadoop/*.env.sh（三个env.sh文件） ####################################################### export JAVA_HOME=/usr/hadoop/jdk
创建exclude文件，用于以后下线hadoop节点 touch /usr/hadoop/hadoop/etc/hadoop/exclude
同步hadoop工程到hadoop002~005机器上面 for ip in `seq 140 145`;do scp -r /usr/hadoop/hadoop slave$ip:/usr/hadoop/;done
修改nn2配置文件yarn-site.xml ##################################################### <property> <name>yarn.resourcemanager.ha.id</name> <value>rm2</value> </property> #####################################################
