部署一个完整的K8S集群(上)

备注：红色字迹为需要在shell中输入的内容：

机器规划：

开机自动关闭防火墙  

[root@k8s-etcd1 ~]#   systemctl disable firewalld.service

关闭防火墙

[root@k8s-etcd1 ~]#   systemctl stop firewalld.service

计划任务同步时间：

0 */1 * * * /usr/sbin/ntpdate cn.pool.ntp.org > /dev/null 2>&1

关闭selinux

[root@k8s-etcd1 ~]#   sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/selinux/config  

[root@k8s-etcd1 ~]#   sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/sysconfig/selinux

 修改主机名

[root@k8s-etcd1 ~]#   hostnamectl set-hostname k8s-etcd1

同步时间

[root@k8s-etcd1 ~]#   sudo timedatectl set-timezone Asia/Shanghai

[root@k8s-etcd1 ~]#   date

[root@k8s-etcd1 ~]#   sudo timedatectl set-local-rtc 0

[root@k8s-etcd1 ~]#   sudo systemctl restart rsyslog

[root@k8s-etcd1 ~]#   sudo systemctl restart crond

关闭swap分区

[root@k8s-etcd1 ~]#   sudo swapoff -a

[root@k8s-etcd1 ~]#   free -m

              total        used        free      shared  buff/cache   available

Mem:           3770         136        3510          11         124        3449

Swap:             0           0           0                确认swap分区已关闭

开机自动关闭swap分区

[root@k8s-etcd1 ~]#   sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

将主机解析到hosts文件里面

[root@k8s-etcd1 ~]#   cat >> /etc/hosts << EOF

192.168.1.121   k8s-etcd1

192.168.1.122   k8s-etcd2

192.168.1.123   k8s-etcd3

192.168.1.124   k8s-master1

192.168.1.125   k8s-master2

192.168.1.126   k8s-master3

192.168.1.127   k8s-node1

192.168.1.128   k8s-node2

192.168.1.129   k8s-node3

192.168.1.130   LVS1

192.168.1.131   LVS2

EOF

环境准备：

软件准备：

[root@k8s-etcd1 src]# ls

etcd.tar.gz  HA.zip  k8s-master.tar.gz  k8s-node.tar.gz  TLS.tar.gz

[root@k8s-etcd1 src]# tar xf etcd.tar.gz 

[root@k8s-etcd1 src]# ls

etcd  etcd.service  etcd.tar.gz  HA.zip  k8s-master.tar.gz  k8s-node.tar.gz  TLS.tar.gz

[root@k8s-etcd1 src]# mv etcd /opt/

[root@k8s-etcd1 src]# ls

etcd.service  etcd.tar.gz  HA.zip  k8s-master.tar.gz  k8s-node.tar.gz  TLS.tar.gz

[root@k8s-etcd1 src]# cp etcd.service /usr/lib/systemd/system/

[root@k8s-etcd1 src]# ls

etcd.service  etcd.tar.gz  HA.zip  k8s-master.tar.gz  k8s-node.tar.gz  TLS.tar.gz

[root@k8s-etcd1 src]# tar xf TLS.tar.gz 

[root@k8s-etcd1 src]# ls

etcd.service  etcd.tar.gz  HA.zip  k8s-master.tar.gz  k8s-node.tar.gz  TLS  TLS.tar.gz

[root@k8s-etcd1 src]# cd TLS

[root@k8s-etcd1 TLS]# ls

cfssl  cfssl-certinfo  cfssljson  cfssl.sh  etcd  k8s

[root@k8s-etcd1 TLS]# vim cfssl.sh 

#curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl

#curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson

#curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo

cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin

chmod +x /usr/local/bin/cfssl*

[root@k8s-etcd1 TLS]# cd etcd/

[root@k8s-etcd1 etcd]# ls

ca-config.json  ca-csr.json  generate_etcd_cert.sh  server-csr.json

[root@k8s-etcd1 etcd]# vim server-csr.json 

{

    "CN": "etcd",

    "hosts": [

        "192.168.1.121",

        "192.168.1.122",

        "192.168.1.123"

        ],

    "key": {

        "algo": "rsa",

        "size": 2048

    },

    "names": [

        {

            "C": "CN",

            "L": "BeiJing",

            "ST": "BeiJing"

        }

    ]

}

[root@k8s-etcd1 etcd]# cat generate_etcd_cert.sh 

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

[root@k8s-etcd1 TLS]# bash -x cfssl.sh 

+ cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin

+ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssl-certinfo /usr/local/bin/cfssljson

[root@k8s-etcd1 TLS]# cd etcd/

[root@k8s-etcd1 etcd]# ls

ca-config.json  ca-csr.json  generate_etcd_cert.sh  server-csr.json

[root@k8s-etcd1 etcd]# ./generate_etcd_cert.sh 

2020/01/29 17:16:06 [INFO] generating a new CA key and certificate from CSR

2020/01/29 17:16:06 [INFO] generate received request

2020/01/29 17:16:06 [INFO] received CSR

2020/01/29 17:16:06 [INFO] generating key: rsa-2048

2020/01/29 17:16:07 [INFO] encoded CSR

2020/01/29 17:16:07 [INFO] signed certificate with serial number 269437542636059925584584985232691741175950103574

2020/01/29 17:16:08 [INFO] generate received request

2020/01/29 17:16:08 [INFO] received CSR

2020/01/29 17:16:08 [INFO] generating key: rsa-2048

2020/01/29 17:16:08 [INFO] encoded CSR

2020/01/29 17:16:09 [INFO] signed certificate with serial number 193231136641378501576889547197718117578468352944

2020/01/29 17:16:09 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for

websites. For more information see the Baseline Requirements for the Issuance and Management

of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);

specifically, section 10.2.3 ("Information Requirements").

[root@k8s-etcd1 etcd]# ls

ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  generate_etcd_cert.sh  server.csr  server-csr.json  server-key.pem  server.pem

[root@k8s-etcd1 etcd]# cp ca.pem server-key.pem server.pem /opt/etcd/ssl/

cp: overwrite ‘/opt/etcd/ssl/ca.pem’? y

cp: overwrite ‘/opt/etcd/ssl/server-key.pem’? y

cp: overwrite ‘/opt/etcd/ssl/server.pem’? y

[root@k8s-etcd1 ssl]# vim /opt/etcd/cfg/etcd.conf 

#[Member]

ETCD_NAME="etcd-1"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

ETCD_LISTEN_PEER_URLS="https://192.168.1.121:2380"

ETCD_LISTEN_CLIENT_URLS="https://192.168.1.121:2379"

#[Clustering]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.121:2380"

ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.121:2379"

ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"

ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"

ETCD_INITIAL_CLUSTER_STATE="new"

 "/opt/etcd/cfg/etcd.conf" 13L, 510C written

复制配置文件和etcd程序到另外两个节点：

[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.123:/opt/

The authenticity of host '192.168.1.123 (192.168.1.123)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.123' (ECDSA) to the list of known hosts.

root@192.168.1.123's password: 

etcd                                                                                                                                   100%   16MB  10.3MB/s   00:01    

etcdctl                                                                                                                                100%   13MB  12.9MB/s   00:01    

etcd.conf                                                                                                                              100%  510    88.3KB/s   00:00    

ca.pem                                                                                                                                 100% 1265   263.3KB/s   00:00    

server.pem                                                                                                                             100% 1338   267.5KB/s   00:00    

server-key.pem                                                                                                                         100% 1675    51.1KB/s   00:00    

[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.122:/opt/

root@192.168.1.122's password: 

etcd                                                                                                                                   100%   16MB  11.7MB/s   00:01    

etcdctl                                                                                                                                100%   13MB   9.0MB/s   00:01    

etcd.conf                                                                                                                              100%  510    94.0KB/s   00:00    

ca.pem                                                                                                                                 100% 1265   264.9KB/s   00:00    

server.pem                                                                                                                             100% 1338   295.0KB/s   00:00    

server-key.pem                                                                                                                         100% 1675   140.1KB/s   00:00    

[root@k8s-etcd2 cfg]# vim etcd.conf 

#[Member]

ETCD_NAME="etcd-2"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

ETCD_LISTEN_PEER_URLS="https://192.168.1.122:2380"

ETCD_LISTEN_CLIENT_URLS="https://192.168.1.122:2379"

#[Clustering]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.122:2380"

ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.122:2379"

ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"

ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"

ETCD_INITIAL_CLUSTER_STATE="new"

[root@k8s-etcd3 cfg]# cat etcd.conf 

#[Member]

ETCD_NAME="etcd-3"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

ETCD_LISTEN_PEER_URLS="https://192.168.1.123:2380"

ETCD_LISTEN_CLIENT_URLS="https://192.168.1.123:2379"

#[Clustering]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.123:2380"

ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.123:2379"

ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"

ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"

ETCD_INITIAL_CLUSTER_STATE="new"

[root@k8s-etcd1 src]# scp -r etcd.service root@192.168.1.122:/usr/lib/systemd/system/

root@192.168.1.122's password: 

etcd.service                                                                                                                           100% 1078   202.6KB/s   00:00    

[root@k8s-etcd1 src]# scp -r etcd.service root@192.168.1.123:/usr/lib/systemd/system/

root@192.168.1.123's password: 

etcd.service                                                                                                                           100% 1078   209.8KB/s   00:00  

启动etcd服务并加入开机自启动：

[root@k8s-etcd1 src]# systemctl restart etcd.service

[root@k8s-etcd1 src]# systemctl status etcd

● etcd.service - Etcd Server

   Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 17:33:30 CST; 11s ago

 Main PID: 13531 (etcd)

   CGroup: /system.slice/etcd.service

           └─13531 /opt/etcd/bin/etcd --name=etcd-1 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.121:2380 --listen-client-urls=https://1...

Jan 29 17:33:30 k8s-etcd1 etcd[13531]: enabled capabilities for version 3.0

Jan 29 17:33:32 k8s-etcd1 etcd[13531]: peer c48307bcc0ac155e became active

Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 reader)

Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 writer)

Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message writer)

Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message reader)

Jan 29 17:33:34 k8s-etcd1 etcd[13531]: updating the cluster version from 3.0 to 3.3

Jan 29 17:33:35 k8s-etcd1 etcd[13531]: updated the cluster version from 3.0 to 3.3

Jan 29 17:33:35 k8s-etcd1 etcd[13531]: enabled capabilities for version 3.3

Jan 29 17:33:35 k8s-etcd1 etcd[13531]: health check for peer c48307bcc0ac155e could not connect: dial tcp 192.168.1.123:2380: connect: connection refused (p..._MESSAGE")

Hint: Some lines were ellipsized, use -l to show in full.

[root@k8s-etcd1 src]# systemctl enable etcd

Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.

[root@k8s-etcd2 cfg]# systemctl restart etcd.service

[root@k8s-etcd2 cfg]# systemctl status etcd

● etcd.service - Etcd Server

   Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 17:33:30 CST; 34s ago

 Main PID: 13101 (etcd)

   CGroup: /system.slice/etcd.service

           └─13101 /opt/etcd/bin/etcd --name=etcd-2 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.122:2380 --listen-client-urls=https://1...

Jan 29 17:33:30 k8s-etcd2 etcd[13101]: set the initial cluster version to 3.0

Jan 29 17:33:30 k8s-etcd2 etcd[13101]: enabled capabilities for version 3.0

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: peer c48307bcc0ac155e became active

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 writer)

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message writer)

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 reader)

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: b5900dafc7a3b5f9 initialzed peer connection; fast-forwarding 8 ticks (election ticks 10) with 2 active peer(s)

Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message reader)

Jan 29 17:33:35 k8s-etcd2 etcd[13101]: updated the cluster version from 3.0 to 3.3

Jan 29 17:33:35 k8s-etcd2 etcd[13101]: enabled capabilities for version 3.3

[root@k8s-etcd2 cfg]# systemctl enable etcd

Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.

[root@k8s-etcd3 cfg]# systemctl restart etcd.service

[root@k8s-etcd3 cfg]# systemctl status etcd

● etcd.service - Etcd Server

   Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 17:33:32 CST; 36s ago

 Main PID: 13071 (etcd)

   CGroup: /system.slice/etcd.service

           └─13071 /opt/etcd/bin/etcd --name=etcd-3 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.123:2380 --listen-client-urls=https://1...

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: set the initial cluster version to 3.0

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: enabled capabilities for version 3.0

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: published {Name:etcd-3 ClientURLs:[https://192.168.1.123:2379]} to cluster ff7100b1c25258f1

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: ready to serve client requests

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: serving insecure client requests on 127.0.0.1:2379, this is strongly discouraged!

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: ready to serve client requests

Jan 29 17:33:32 k8s-etcd3 systemd[1]: Started Etcd Server.

Jan 29 17:33:32 k8s-etcd3 etcd[13071]: serving client requests on 192.168.1.123:2379

Jan 29 17:33:35 k8s-etcd3 etcd[13071]: updated the cluster version from 3.0 to 3.3

Jan 29 17:33:35 k8s-etcd3 etcd[13071]: enabled capabilities for version 3.3

[root@k8s-etcd3 cfg]# systemctl enable etcd

Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.

至此，etcd集群安装完毕

安装k8s-master集群开始：

将k8s-master.tar.gz软件包复制到三个master节点：

[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.124:/usr/local/src/

The authenticity of host '192.168.1.124 (192.168.1.124)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.124' (ECDSA) to the list of known hosts.

root@192.168.1.124's password: 

k8s-master.tar.gz                                                                                                                      100%   87MB  19.2MB/s   00:04    

[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.125:/usr/local/src/

The authenticity of host '192.168.1.125 (192.168.1.125)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.125' (ECDSA) to the list of known hosts.

root@192.168.1.125's password: 

k8s-master.tar.gz                                                                                                                      100%   87MB  17.3MB/s   00:05    

[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.126:/usr/local/src/

The authenticity of host '192.168.1.126 (192.168.1.126)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.126' (ECDSA) to the list of known hosts.

root@192.168.1.126's password: 

k8s-master.tar.gz                                                                                                                      100%   87MB  15.2MB/s   00:05  

[root@k8s-etcd1 src]# scp -r TLS.tar.gz root@192.168.1.124:/usr/local/src/

root@192.168.1.124's password: 

TLS.tar.gz                                                                                                                             100% 5715KB  15.5MB/s   00:00 

[root@k8s-master1 src]# tar xf TLS.tar.gz 

[root@k8s-master1 TLS]# ls

cfssl  cfssl-certinfo  cfssljson  cfssl.sh  etcd  k8s

[root@k8s-master1 TLS]# cd k8s/

[root@k8s-master1 k8s]# ls

ca-config.json  ca-csr.json  generate_k8s_cert.sh  kube-proxy-csr.json  server-csr.json

[root@k8s-master1 src]# cat TLS/k8s/server-csr.json 

{

    "CN": "kubernetes",

    "hosts": [

      "10.0.0.1",

      "127.0.0.1",

      "kubernetes",

      "kubernetes.default",

      "kubernetes.default.svc",

      "kubernetes.default.svc.cluster",

      "kubernetes.default.svc.cluster.local",

      "192.168.1.120",

      "192.168.1.121",

      "192.168.1.122",

      "192.168.1.123",

      "192.168.1.124",

      "192.168.1.125",

      "192.168.1.126",

      "192.168.1.127",

      "192.168.1.128",

      "192.168.1.129",

      "192.168.1.130",

      "192.168.1.131",

      "192.168.1.132",

      "192.168.1.133"

    ],

    "key": {

        "algo": "rsa",

        "size": 2048

    },

    "names": [

        {

            "C": "CN",

            "L": "BeiJing",

            "ST": "BeiJing",

            "O": "k8s",

            "OU": "System"

        }

    ]

}

[root@k8s-master1 src]# cd TLS

[root@k8s-master1 TLS]# ls

cfssl  cfssl-certinfo  cfssljson  cfssl.sh  etcd  k8s

[root@k8s-master1 TLS]# bash -x cfssl.sh 

+ cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin

+ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssl-certinfo /usr/local/bin/cfssljson

[root@k8s-master1 TLS]# cd k8s/

[root@k8s-master1 k8s]# ls

ca-config.json  ca-csr.json  generate_k8s_cert.sh  kube-proxy-csr.json  server-csr.json

[root@k8s-master1 k8s]# ./generate_k8s_cert.sh 

2020/01/29 19:48:44 [INFO] generating a new CA key and certificate from CSR

2020/01/29 19:48:44 [INFO] generate received request

2020/01/29 19:48:44 [INFO] received CSR

2020/01/29 19:48:44 [INFO] generating key: rsa-2048

2020/01/29 19:48:46 [INFO] encoded CSR

2020/01/29 19:48:46 [INFO] signed certificate with serial number 644734139928112787467618196045542040345765034825

2020/01/29 19:48:46 [INFO] generate received request

2020/01/29 19:48:46 [INFO] received CSR

2020/01/29 19:48:46 [INFO] generating key: rsa-2048

2020/01/29 19:48:49 [INFO] encoded CSR

2020/01/29 19:48:49 [INFO] signed certificate with serial number 641751437813294970674608941868096756464221965513

2020/01/29 19:48:49 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for

websites. For more information see the Baseline Requirements for the Issuance and Management

of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);

specifically, section 10.2.3 ("Information Requirements").

2020/01/29 19:48:49 [INFO] generate received request

2020/01/29 19:48:49 [INFO] received CSR

2020/01/29 19:48:49 [INFO] generating key: rsa-2048

2020/01/29 19:48:51 [INFO] encoded CSR

2020/01/29 19:48:51 [INFO] signed certificate with serial number 573406333537228053382286849722926605568836533693

2020/01/29 19:48:51 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for

websites. For more information see the Baseline Requirements for the Issuance and Management

of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);

specifically, section 10.2.3 ("Information Requirements").

[root@k8s-master1 k8s]# ls

ca-config.json  ca-csr.json  ca.pem                kube-proxy.csr       kube-proxy-key.pem  server.csr       server-key.pem

ca.csr          ca-key.pem   generate_k8s_cert.sh  kube-proxy-csr.json  kube-proxy.pem      server-csr.json  server.pem

k8s-master集群需要用到etcd的ssl验证文件。所以要将etcd节点上面的etcd目录拷贝到k8s-master下面的/opt目录下，这是有kubernetes的配置文件决定的。

[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.124:/opt/

root@192.168.1.124's password: 

etcd                                                                                                                                   100%   16MB  16.3MB/s   00:00    

etcdctl                                                                                                                                100%   13MB  19.6MB/s   00:00    

etcd.conf                                                                                                                              100%  510   103.5KB/s   00:00    

ca.pem                                                                                                                                 100% 1265   424.0KB/s   00:00    

server.pem                                                                                                                             100% 1338   426.6KB/s   00:00    

server-key.pem                                                                                                                         100% 1675   441.4KB/s   00:00    

[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.125:/opt/

root@192.168.1.125's password: 

etcd                                                                                                                                   100%   16MB  16.4MB/s   00:00    

etcdctl                                                                                                                                100%   13MB  12.9MB/s   00:01    

etcd.conf                                                                                                                              100%  510    92.4KB/s   00:00    

ca.pem                                                                                                                                 100% 1265   273.2KB/s   00:00    

server.pem                                                                                                                             100% 1338   202.1KB/s   00:00    

server-key.pem                                                                                                                         100% 1675   394.4KB/s   00:00    

[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.126:/opt/

root@192.168.1.126's password: 

etcd                                                                                                                                   100%   16MB  16.1MB/s   00:01    

etcdctl                                                                                                                                100%   13MB  12.9MB/s   00:01    

etcd.conf                                                                                                                              100%  510    84.2KB/s   00:00    

ca.pem                                                                                                                                 100% 1265   348.9KB/s   00:00    

server.pem                                                                                                                             100% 1338   251.2KB/s   00:00    

server-key.pem                                                                                                                         100% 1675   213.2KB/s   00:00  

[root@k8s-master1 src]# ls

k8s-master.tar.gz  kube-apiserver.service  kube-controller-manager.service  kubernetes  kube-scheduler.service

[root@k8s-master1 src]# \cp -r kubernetes /opt/

将生成的配置文件拷贝到/opt/kubernetes/ssl/目录下

[root@k8s-master1 k8s]# cp ca-key.pem  ca.pem  server-key.pem  server.pem /opt/kubernetes/ssl/

  : overwrite ‘/opt/kubernetes/ssl/ca-key.pem’? y

cp: overwrite ‘/opt/kubernetes/ssl/ca.pem’? y

[root@k8s-master1 k8s]# cd /opt/kubernetes/

[root@k8s-master1 kubernetes]# ls

bin  cfg  logs  ssl

[root@k8s-master1 kubernetes]# cd ssl/

[root@k8s-master1 ssl]# ls

ca-key.pem  ca.pem  server-key.pem  server.pem

[root@k8s-master1 ssl]# ll

total 16

-rw------- 1 root root 1679 Jan 29 20:00 ca-key.pem

-rw-r--r-- 1 root root 1359 Jan 29 20:00 ca.pem

-rw------- 1 root root 1679 Jan 29 20:00 server-key.pem

-rw-r--r-- 1 root root 1716 Jan 29 20:00 server.pem

修改Kubernetes的配置文件kube-apiserver.conf 

[root@k8s-master1 cfg]# cat kube-apiserver.conf 

KUBE_APISERVER_OPTS="--logtostderr=false \

--v=2 \

--log-dir=/opt/kubernetes/logs \

--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \    #etcd集群的地址和端口

--bind-address=192.168.1.124 \                                                                                                               #本机地址

--secure-port=6443 \

--advertise-address=192.168.1.124 \                                                                                                        #本机地址

--allow-privileged=true \

--service-cluster-ip-range=10.0.0.0/24 \

--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \

--authorization-mode=RBAC,Node \

--enable-bootstrap-token-auth=true \

--token-auth-file=/opt/kubernetes/cfg/token.csv \

--service-node-port-range=30000-32767 \

--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \

--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \

--tls-cert-file=/opt/kubernetes/ssl/server.pem  \

--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \

--client-ca-file=/opt/kubernetes/ssl/ca.pem \

--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \

--etcd-cafile=/opt/etcd/ssl/ca.pem \

--etcd-certfile=/opt/etcd/ssl/server.pem \

--etcd-keyfile=/opt/etcd/ssl/server-key.pem \

--audit-log-maxage=30 \

--audit-log-maxbackup=3 \

--audit-log-maxsize=100 \

--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"

[root@k8s-master1 opt]# scp -r kubernetes root@192.168.1.125:/opt/

root@192.168.1.125's password: 

kubectl                                                                                                                                100%   45MB  16.1MB/s   00:02    

kube-apiserver                                                                                                                         100%  167MB  18.5MB/s   00:09    

kube-controller-manager                                                                                                                100%  115MB  14.4MB/s   00:08    

kube-scheduler                                                                                                                         100%   43MB  12.5MB/s   00:03    

token.csv                                                                                                                              100%   83    20.2KB/s   00:00    

kube-controller-manager.conf                                                                                                           100%  546    98.2KB/s   00:00    

kube-scheduler.conf                                                                                                                    100%  148    20.4KB/s   00:00    

kube-apiserver.conf                                                                                                                    100% 1193   252.7KB/s   00:00    

ca-key.pem                                                                                                                             100% 1679   337.2KB/s   00:00    

ca.pem                                                                                                                                 100% 1359   297.2KB/s   00:00    

server-key.pem                                                                                                                         100% 1679   289.3KB/s   00:00    

server.pem                                                                                                                             100% 1716   292.5KB/s   00:00    

[root@k8s-master1 opt]# scp -r kubernetes root@192.168.1.126:/opt/

root@192.168.1.126's password: 

kubectl                                                                                                                                100%   45MB  14.8MB/s   00:03    

kube-apiserver                                                                                                                         100%  167MB  16.7MB/s   00:10    

kube-controller-manager                                                                                                                100%  115MB  14.4MB/s   00:08    

kube-scheduler                                                                                                                         100%   43MB  14.4MB/s   00:02    

token.csv                                                                                                                              100%   83    22.3KB/s   00:00    

kube-controller-manager.conf                                                                                                           100%  546    95.2KB/s   00:00    

kube-scheduler.conf                                                                                                                    100%  148    35.0KB/s   00:00    

kube-apiserver.conf                                                                                                                    100% 1193   176.9KB/s   00:00    

ca-key.pem                                                                                                                             100% 1679   340.2KB/s   00:00    

ca.pem                                                                                                                                 100% 1359   266.1KB/s   00:00    

server-key.pem                                                                                                                         100% 1679   298.4KB/s   00:00    

server.pem                                                                                                                             100% 1716   331.6KB/s   00:00 

修改k8s-master2 的配置文件kube-apiserver.conf 

[root@k8s-master2 cfg]# cat kube-apiserver.conf 

KUBE_APISERVER_OPTS="--logtostderr=false \

--v=2 \

--log-dir=/opt/kubernetes/logs \

--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \

--bind-address=192.168.1.125 \

--secure-port=6443 \

--advertise-address=192.168.1.125 \

--allow-privileged=true \

--service-cluster-ip-range=10.0.0.0/24 \

--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \

--authorization-mode=RBAC,Node \

--enable-bootstrap-token-auth=true \

--token-auth-file=/opt/kubernetes/cfg/token.csv \

--service-node-port-range=30000-32767 \

--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \

--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \

--tls-cert-file=/opt/kubernetes/ssl/server.pem  \

--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \

--client-ca-file=/opt/kubernetes/ssl/ca.pem \

--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \

--etcd-cafile=/opt/etcd/ssl/ca.pem \

--etcd-certfile=/opt/etcd/ssl/server.pem \

--etcd-keyfile=/opt/etcd/ssl/server-key.pem \

--audit-log-maxage=30 \

--audit-log-maxbackup=3 \

--audit-log-maxsize=100 \

--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"

修改k8s-master3 的配置文件kube-apiserver.conf 

[root@k8s-master3 cfg]# cat kube-apiserver.conf 

KUBE_APISERVER_OPTS="--logtostderr=false \

--v=2 \

--log-dir=/opt/kubernetes/logs \

--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \

--bind-address=192.168.1.126 \

--secure-port=6443 \

--advertise-address=192.168.1.126 \

--allow-privileged=true \

--service-cluster-ip-range=10.0.0.0/24 \

--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \

--authorization-mode=RBAC,Node \

--enable-bootstrap-token-auth=true \

--token-auth-file=/opt/kubernetes/cfg/token.csv \

--service-node-port-range=30000-32767 \

--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \

--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \

--tls-cert-file=/opt/kubernetes/ssl/server.pem  \

--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \

--client-ca-file=/opt/kubernetes/ssl/ca.pem \

--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \

--etcd-cafile=/opt/etcd/ssl/ca.pem \

--etcd-certfile=/opt/etcd/ssl/server.pem \

--etcd-keyfile=/opt/etcd/ssl/server-key.pem \

--audit-log-maxage=30 \

--audit-log-maxbackup=3 \

--audit-log-maxsize=100 \

--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"

[root@k8s-master1 src]# cp *.service /usr/lib/systemd/system/

[root@k8s-master1 src]# scp -r  *.service root@192.168.1.125:/usr/lib/systemd/system/

The authenticity of host '192.168.1.125 (192.168.1.125)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.125' (ECDSA) to the list of known hosts.

root@192.168.1.125's password: 

kube-apiserver.service                                                                                                                 100%  286    67.9KB/s   00:00    

kube-controller-manager.service                                                                                                        100%  321    68.1KB/s   00:00    

kube-scheduler.service                                                                                                                 100%  285    46.3KB/s   00:00    

[root@k8s-master1 src]# scp -r  *.service root@192.168.1.126:/usr/lib/systemd/system/

The authenticity of host '192.168.1.126 (192.168.1.126)' can't be established.

ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.

ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.

Are you sure you want to continue connecting (yes/no)? yes

Warning: Permanently added '192.168.1.126' (ECDSA) to the list of known hosts.

root@192.168.1.126's password: 

kube-apiserver.service                                                                                                                 100%  286    57.5KB/s   00:00    

kube-controller-manager.service                                                                                                        100%  321    64.6KB/s   00:00    

kube-scheduler.service                                                                                                                 100%  285    71.7KB/s   00:00    

3台机器都将服务加入开机自启动：

[root@k8s-master1 opt]#  systemctl enable kube-apiserver

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.

[root@k8s-master1 opt]# systemctl enable kube-controller-manager

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.

[root@k8s-master1 opt]# systemctl enable kube-scheduler

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.

[root@k8s-master2 opt]#  systemctl enable kube-apiserver

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.

[root@k8s-master2 opt]# systemctl enable kube-controller-manager

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.

[root@k8s-master2 opt]# systemctl enable kube-scheduler

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.

[root@k8s-master3 opt]#  systemctl enable kube-apiserver

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.

[root@k8s-master3 opt]# systemctl enable kube-controller-manager

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.

[root@k8s-master3 opt]# systemctl enable kube-scheduler

Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.

[root@k8s-master1 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/

[root@k8s-master2 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/

[root@k8s-master3 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/

启动服务并验证是否启动成功，如出现active (running)则代表启动成功

[root@k8s-master1 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler

[root@k8s-master1 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler

● kube-apiserver.service - Kubernetes API Server

   Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 6s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12430 (kube-apiserver)

   CGroup: /system.slice/kube-apiserver.service

           └─12430 /opt/kubernetes/bin/kube-apiserver --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --etcd-servers=https://192.168.1.121:2379,https://192.1...

Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes API Server.

● kube-controller-manager.service - Kubernetes Controller Manager

   Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12437 (kube-controller)

   CGroup: /system.slice/kube-controller-manager.service

           └─12437 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect=true --master=127.0.0.1:8080 --...

Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes Controller Manager.

Jan 29 20:26:02 k8s-master1 kube-controller-manager[12437]: Flag --address has been deprecated, see --bind-address instead.

● kube-scheduler.service - Kubernetes Scheduler

   Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12450 (kube-scheduler)

   CGroup: /system.slice/kube-scheduler.service

           └─12450 /opt/kubernetes/bin/kube-scheduler --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --master=127.0.0.1:8080 --address=127.0....

Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes Scheduler.

[root@k8s-master2 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler

[root@k8s-master2 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler

● kube-apiserver.service - Kubernetes API Server

   Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 6s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12430 (kube-apiserver)

   CGroup: /system.slice/kube-apiserver.service

           └─12430 /opt/kubernetes/bin/kube-apiserver --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --etcd-servers=https://192.168.1.121:2379,https://192.1...

Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes API Server.

● kube-controller-manager.service - Kubernetes Controller Manager

   Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12437 (kube-controller)

   CGroup: /system.slice/kube-controller-manager.service

           └─12437 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect=true --master=127.0.0.1:8080 --...

Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes Controller Manager.

Jan 29 20:26:02 k8s-master2 kube-controller-manager[12437]: Flag --address has been deprecated, see --bind-address instead.

● kube-scheduler.service - Kubernetes Scheduler

   Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)

   Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago

     Docs: https://github.com/kubernetes/kubernetes

 Main PID: 12450 (kube-scheduler)

   CGroup: /system.slice/kube-scheduler.service

           └─12450 /opt/kubernetes/bin/kube-scheduler --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --master=127.0.0.1:8080 --address=127.0....

Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes Scheduler.

[root@k8s-master3 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler

[root@k8s-master3 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler

● kube-apiserver.service - Kubernetes API Server

   Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.serv