在 Debian 上搭建 HDFS 集群的实操指南
一 环境规划与准备
192.168.50.201 node1
192.168.50.202 node2
192.168.50.203 node3
二 安装 Java 与 Hadoop
sudo apt update
sudo apt install -y openjdk-8-jdk
java -version
wget -O /tmp/hadoop-3.3.6.tar.gz \
https://dlcdn.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz
sudo mkdir -p /usr/local
sudo tar -xzf /tmp/hadoop-3.3.6.tar.gz -C /usr/local
sudo ln -sfn /usr/local/hadoop-3.3.6 /usr/local/hadoop
cat >/etc/profile.d/hadoop.sh <<'EOF'
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
EOF
source /etc/profile
hadoop version
三 配置 HDFS 核心文件
cat >$HADOOP_HOME/etc/hadoop/hadoop-env.sh <<'EOF'
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
# 如使用 root 运行,请按需设置(生产建议专用 hadoop 用户)
export HDFS_NAMENODE_USER=hadoop
export HDFS_DATANODE_USER=hadoop
export HDFS_SECONDARYNAMENODE_USER=hadoop
EOF
cat >$HADOOP_HOME/etc/hadoop/core-site.xml <<'EOF'
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://node1:8020</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/export/data/hadoop</value>
</property>
<property>
<name>hadoop.http.staticuser.user</name>
<value>hadoop</value>
</property>
</configuration>
EOF
cat >$HADOOP_HOME/etc/hadoop/hdfs-site.xml <<'EOF'
<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///export/data/hadoop/hdfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///export/data/hadoop/hdfs/data</value>
</property>
<!-- 可选:启用 WebHDFS -->
<!--
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
-->
</configuration>
EOF
cat >$HADOOP_HOME/etc/hadoop/workers <<'EOF'
node2
node3
EOF
sudo mkdir -p /export/data/hadoop/hdfs/{name,data}
sudo chown -R hadoop:hadoop /export/data/hadoop
四 启动与验证
sudo -u hadoop hdfs namenode -format
# 在 NameNode 执行
sudo -u hadoop start-dfs.sh
# 各节点应出现相应进程:NameNode/DataNode/SecondaryNameNode
jps
# 查看 HDFS 报告
sudo -u hadoop hdfs dfsadmin -report
# 创建测试目录并写入文件
sudo -u hadoop hdfs dfs -mkdir -p /user/hadoop
echo "Hello HDFS" | sudo -u hadoop hdfs dfs -put - /user/hadoop/hello.txt
sudo -u hadoop hdfs dfs -cat /user/hadoop/hello.txt
五 常见问题与优化