在centos7.4环境下安装用cm5.16安装CDH

修改ip 然后重启/etc/init.d/netwrok restart

PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=no
IPV6_AUTOCONF=no
IPV6_DEFROUTE=no
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
UUID=a0ad438d-a586-4c30-b624-601b342a39c4
DEVICE=ens32
ONBOOT=yes
NETMASK=255.255.255.0
IPADDR=192.168.52.130
GATEWAY=192.168.52.2
DNS1=223.5.5.5

修改主机名

hostnamectl set-hostname bigdata-cdh01.kfk.com

修改本机DNS /etc/host

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
#::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.52.130 bigdata-cdh01.kfk.com bigdata-cdh01
192.168.52.131 bigdata-cdh02.kfk.com bigdata-cdj02
192.168.52.132 bigdata-cdh03.kfk.com bigdata-cdh03

关闭防火墙

systemctl stop firewalld.service
systemctl disable firewalld.service （开机不启动）
systemctl status firewalld.service

禁用selinux cat /etc/sysconfig/selinux

SELINUX=disabled

临时关闭selinux

getenforce
setenforce 0
getenforce

禁用ipv6

echo "            " >> /etc/modprobe.d/dist.conf
echo "alias net-pf-10 off" >> /etc/modprobe.d/dist.conf
echo "alias ipv6 off" >> /etc/modprobe.d/dist.conf

卸载JDK

rpm -qa | grep java
rpm -e --nodeps xxx

设置文件打开数目和用户最大进程数
文件打开数目

ulimit -a

用户最大进程数

ulimit -u

设置 /etc/security/limits.conf

*    soft   nofile 32768
*    hard   nofile 1048576
*    soft   nproc  65536
*    hard   nporc  unlimited
*    soft   memlock unlimited
*    hard   memlock unlimited

ssh互信

ssh-keygen -t rsa
touch authorized_keys
chmod 600 authorized_keys
cat id_rsa.pub > authorized_keys
ssh-copy-id -i id_rsa.pub root@192.168.52.130

使用ntp进行时间同步
主节点

driftfile /var/lib/ntp/drift
restrict 192.168.52.130 nomodify notrap nopeer noquery    
restrict 127.0.0.1
restrict ::1
restrict 192.168.52.2 mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge  127.127.1.0 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor

从节点，其他节点同理

driftfile /var/lib/ntp/drift
restrict 192.168.52.131 nomodify notrap nopeer noquery    
restrict 127.0.0.1
restrict ::1
restrict 192.168.52.2 mask 255.255.255.0 nomodify notrap
server 192.168.52.130
Fudge  192.168.52.130 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor

先关闭ntpd

ntpdate 192.168.52.130
service ntpd restart
systemctl enable ntpd

ntpq -p 检查是否同步，有*表示已经同步

关闭透明大页和交换内存

    echo never > /sys/kernel/mm/transparent_hugepage/defrag  
    echo never > /sys/kernel/mm/transparent_hugepage/enabled   
    echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local  
    echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.local  
    chmod +x /etc/rc.d/rc.local  
    echo "vm.swappiness=1" >> /etc/sysctl.conf  
    sysctl -p    //使配置生效 

搭建yum源

rpm -ivh httpd-2.4.6-80.el7.centos.x86_64.rpm httpd-tools-2.4.6-80.el7.centos.x86_64.rpm
mount /dev/sr0 /mnt
ln -s /mnt centos7
/etc/yum.repos.d
[Centos-7]
name=myrepo
baseurl=http://bigdata-cdh01/centos7/
enabled=1
gpgcheck=0
yum clean all
yum makecache
yum install -y createrepo
cd /var/www/html/cm-5.16
createrepo .
cat myrepo.repo
[myrepo]
name=myrepo
baseurl=http://bigdata-cdh01/cm-5.16/
enabled=1
gpgcheck=0

安装mysql5.7数据库官网下载包（mysql-5.7.13-linux-glibc2.5-x86_64.tar.gz）

groupadd mysql
useradd mysql -s /sbin/nologin -M -g mysql
tar -zxvf mysql-5.7.13-linux-glibc2.5-x86_64.tar.gz 
mv mysql-5.7.13-linux-glibc2.5-x86_64 /usr/local/mysql
chown mysql:mysql -R mysql/
mkdir data
chown -R mysql:mysql data/
yum install -y perl-Data-Dumper.x86_64
yum install libaio*
./bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data/
scp -r support-files/mysql.server /etc/init.d/mysqld
vim /etc/init.d/mysqld 
basedir=/usr/local/mysql
datadir=/usr/local/mysql/dat
编辑 /etc/my.cnf
[mysqld]
user=mysql
datadir=/usr/local/mysql/data
socket=/tmp/mysql.sock
log-error=/usr/local/mysql/data/error.log
pid-file=/usr/local/mysql/data/mysql/mysql.pid
tmpdir=/tmp
设置mysql开机启动
chkconfig mysqld on
修改密码alter user user() identified by "123456";flush privileges;
开启远程登录GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;flush privileges;
查看mysql 的用户名密码select host,user,authentication_string from mysql.user;

创建数据库

create database metastore default character set utf8;
CREATE USER 'hive'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON metastore. * TO 'hive'@'%';
FLUSH PRIVILEGES;

create database hive default character set utf8;
CREATE USER 'hive'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON hive. * TO 'hive'@'%';
FLUSH PRIVILEGES;

create database cm default character set utf8;
CREATE USER 'cm'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON cm. * TO 'cm'@'%';
FLUSH PRIVILEGES;

create database am default character set utf8;
CREATE USER 'am'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON am. * TO 'am'@'%';
FLUSH PRIVILEGES;

create database rm default character set utf8;
CREATE USER 'rm'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON rm. * TO 'rm'@'%';
FLUSH PRIVILEGES;

create database hue default character set utf8;
CREATE USER 'hue'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON hue. * TO 'hue'@'%';
FLUSH PRIVILEGES;

create database oozie default character set utf8;
CREATE USER 'oozie'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON oozie. * TO 'oozie'@'%';
FLUSH PRIVILEGES;

拷贝jdbc的包，所有节点都需要安装

cd /usr/share/java/
ln -s mysql-connector-java-5.1.38-bin.jar mysql-connector-java.jar

安装元数据库
/usr/share/cmf/schema/scm_prepare_database.sh mysql cm cm 123456

用命令行安装jdk

yum list | grep oracle
yum install -y oracle-j2sdk1.7.x86_64
export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin

主节点上安装

  yum  install -y cloudera-manager-server cloudera-manager-daemons cloudera-manager-agent

从节点上安装

  yum  install -y cloudera-manager-daemons cloudera-manager-agent

上传CDH文件

CDH-5.16.2-1.cdh6.16.2.p0.8-el7.parcel 
CDH-5.16.2-1.cdh6.16.2.p0.8-el7.parcel.sha1 
manifest.json
mv CDH-5.16.2-1.cdh6.16.2.p0.8-el7.parcel.sha1 CDH-5.16.2-1.cdh6.16.2.p0.8-el7.parcel.sha

启动

  systemctl start cloudera-scm-server

不选


添加服务







kafka安装

http://archive.cloudera.com/kafka/parcels/3.1.1/
KAFKA-3.1.1-1.3.1.1.p0.2-el7.parcel
KAFKA-3.1.1-1.3.1.1.p0.2-el7.parcel.sha
manifest.json
上传至 /opt/cloudera/parcel-repo
上传KAFKA-1.2.0.jar  到 /opt/cloudera/csd