Linux下Hadoop备份实操指南
一 备份目标与总体策略
二 数据层备份与恢复
hdfs dfsadmin -allowSnapshot /path/to/dir
hdfs dfs -createSnapshot /path/to/dir snap_20251204
hdfs dfsadmin -listSnapshots /path/to/dir
# 回滚:先删除损坏目录,再从快照拷贝回去
hdfs dfs -rm -r /path/to/dir
hdfs dfs -cp /path/to/dir/.snapshot/snap_20251204 /path/to/dir
hdfs dfs -deleteSnapshot /path/to/dir snap_20251204
hadoop distcp -m 50 hdfs://src-nn:8020/path hdfs://backup-nn:8020/backup/20251204
hdfs dfs -get /path/to/dir /local/backup/20251204
hdfs dfs -put /local/backup/20251204 hdfs://backup-nn:8020/backup/20251204
hdfs dfs -ls /user/hadoop/.Trash/Current
hdfs dfs -mv /user/hadoop/.Trash/Current/file /user/hadoop/file
hdfs fsck /path -files -blocks -locations
hdfs debug recoverLease -path /path/to/file -retries 10
三 元数据与配置备份与恢复
# 进入安全模式
sudo -u hdfs hdfs dfsadmin -safemode enter
# 落盘合并fsimage并重置edits
sudo -u hdfs hdfs dfsadmin -saveNamespace
# 备份元数据目录(路径以实际dfs.namenode.name.dir为准)
sudo -u hdfs rsync -a /dfs/nn/current /backup/nn-$(date +%F)
# 退出安全模式
sudo -u hdfs hdfs dfsadmin -safemode leave
mysqldump -uroot -p --databases metastore > metastore_$(date +%F).sql
mysql -uroot -p < metastore_$(date +%F).sql
tar czvf hdfs-config-$(date +%F).tar.gz /etc/hadoop/conf /usr/local/hadoop/etc/hadoop
四 自动化与调度示例
#!/usr/bin/env bash
set -e
BACKUP_ROOT="hdfs://backup-nn:8020/backup"
SRC="/user/hadoop/data"
DATE=$(date +%F)
LAST=$(date -d '7 days ago' +%F)
hdfs dfs -mkdir -p $BACKUP_ROOT/$DATE
# 首次全量;后续可改为 -update/-diff + -lastmod/-compare
hadoop distcp -m 50 -bandwidth 200 $SRC $BACKUP_ROOT/$DATE
# 清理7天前备份
hdfs dfs -rm -r -skipTrash $BACKUP_ROOT/$LAST
# 可选:校验
hdfs fsck $BACKUP_ROOT/$DATE -files -blocks -locations || echo "FS check completed with warnings"
0 2 * * * /opt/backup/backup_hdfs_distcp.sh >> /var/log/hdfs_backup.log 2>&1
#!/usr/bin/env bash
set -e
HDFS_SRC="/user/hadoop/data"
LOCAL_DST="/backup/hdfs/$(date +%F)"
mkdir -p $LOCAL_DST
hdfs dfs -get $HDFS_SRC $LOCAL_DST
tar czvf $LOCAL_DST.tar.gz -C $LOCAL_DST .
五 策略与注意事项