Use external-dns to auto-bond route53 in EKS

发布时间：2020-05-20 11:15:36 作者：gxjluck
来源：网络阅读：402

Install EKS cluster

How to install eksctl

# eksctl create cluster -f cluster.yaml
 
 
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
 
metadata:
  name: cluster01
  region: ap-northeast-2
 
vpc:
  subnets:
    public:
      ap-northeast-2a: { id: subnet-dbbb4fb2 }
      ap-northeast-2b: { id: subnet-2b4fa650 }
      ap-northeast-2c: { id: subnet-99182fd3 }
    private:
      ap-northeast-2a: { id: subnet-dbbb4fb2 }
      ap-northeast-2b: { id: subnet-2b4fa650 }
      ap-northeast-2c: { id: subnet-99182fd3 }
 
nodeGroups:
  - name: ng01
    labels: { role: workers }
    instanceType: t2.xlarge
    minSize: 2
    maxSize: 8
    volumeSize: 100
    volumeType: gp2
    ami: auto
    amiFamily: Ubuntu1804
    #privateNetworking: true
    ssh:
      publicKeyName: gexj
      #publicKeyPath: ~/.ssh/gexj.pub

Record run result

[ℹ]  using region ap-northeast-2
[✔]  using existing VPC (vpc-1fa75276) and subnets (private:[subnet-2b4fa650 subnet-99182fd3 subnet-dbbb4fb2] public:[subnet-dbbb4fb2 subnet-2b4fa650 subnet-99182fd3])
[!]  custom VPC/subnets will be used; if resulting cluster doesn't function as expected, make sure to review the configuration of VPC/subnets
[ℹ]  nodegroup "ng01" will use "ami-0f37e6cfe5a2e9281" [Ubuntu1804/1.13]
[ℹ]  using EC2 key pair "gexj"
[ℹ]  using Kubernetes version 1.13
[ℹ]  creating EKS cluster "cluster01" in "ap-northeast-2" region
[ℹ]  1 nodegroup (ng01) was included
[ℹ]  will create a CloudFormation stack for cluster itself and 1 nodegroup stack(s)
[ℹ]  if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=ap-northeast-2 --name=cluster01'
[ℹ]  CloudWatch logging will not be enabled for cluster "cluster01" in "ap-northeast-2"
[ℹ]  you can enable it with 'eksctl utils update-cluster-logging --region=ap-northeast-2 --name=cluster01'
[ℹ]  2 sequential tasks: { create cluster control plane "cluster01", create nodegroup "ng01" }
[ℹ]  building cluster stack "eksctl-cluster01-cluster"
[ℹ]  deploying stack "eksctl-cluster01-cluster"
[ℹ]  building nodegroup stack "eksctl-cluster01-nodegroup-ng01"
[ℹ]  deploying stack "eksctl-cluster01-nodegroup-ng01"
[✔]  all EKS cluster resource for "cluster01" had been created
[✔]  saved kubeconfig as "/root/.kube/config"
[ℹ]  adding role "arn:aws:iam::647035961056:role/eksctl-cluster01-nodegroup-ng01-NodeInstanceRole-GDG5Y6EPZ0B8" to auth ConfigMap
[ℹ]  nodegroup "ng01" has 0 node(s)
[ℹ]  waiting for at least 2 node(s) to become ready in "ng01"
[ℹ]  nodegroup "ng01" has 2 node(s)
[ℹ]  node "ip-172-31-25-102.ap-northeast-2.compute.internal" is ready
[ℹ]  node "ip-172-31-9-210.ap-northeast-2.compute.internal" is ready
[ℹ]  kubectl command should work with "/root/.kube/config", try 'kubectl get nodes'
[✔]  EKS cluster "cluster01" in "ap-northeast-2" region is ready

Add IAM Permissions to the roles created above

{
 "Version": "2012-10-17",
 "Statement": [
   {
     "Effect": "Allow",
     "Action": [
       "route53:ChangeResourceRecordSets"
     ],
     "Resource": [
       "arn:aws:route53:::hostedzone/*"
     ]
   },
   {
     "Effect": "Allow",
     "Action": [
       "route53:ListHostedZones",
       "route53:ListResourceRecordSets"
     ],
     "Resource": [
       "*"
     ]
   }
 ]
}

Set up a hosted zone

## If you prefer to try-out ExternalDNS in one of the existing hosted-zones you can skip this step

# aws route53 create-hosted-zone --name "gexj.club." --caller-reference "gexj-clube-$(date +%s)"

{
    "Location": "<a href="https://route53.amazonaws.com/2013-04-01/hostedzone/Z3P062U2BQWNDS" "="" >https://route53.amazonaws.com/2013-04-01/hostedzone/Z3P062U2BQWNDS",
    "HostedZone": {
        "Id": "/hostedzone/Z3P062U2BQWNDS",
        "Name": "gexj.club.",
        "CallerReference": "gexj-clube-1568880027",
        "Config": {
            "PrivateZone": false
        },
        "ResourceRecordSetCount": 2
    },
    "ChangeInfo": {
        "Id": "/change/C1GWAKWYESG64Y",
        "Status": "PENDING",
        "SubmittedAt": "2019-09-19T08:00:28.557Z"
    },
    "DelegationSet": {
        "NameServers": [
            "ns-771.awsdns-32.net",
            "ns-378.awsdns-47.com",
            "ns-1192.awsdns-21.org",
            "ns-1786.awsdns-31.co.uk"
        ]
    }
}

Deploy ExternalDNS

$ kubectl apply -f external-dns-with-rbac.yaml

# kubectl logs external-dns-5454846d9b-sdjzd -f
time="2019-09-19T08:06:43Z" level=info msg="config: {Master: KubeConfig: RequestTimeout:30s IstioIngressGatewayServices:[istio-system/istio-ingressgateway] ContourLoadBalancerService:heptio-contour/contour Sources:[service ingress] Namespace: AnnotationFilter: FQDNTemplate: CombineFQDNAndAnnotation:false IgnoreHostnameAnnotation:false Compatibility: PublishInternal:false PublishHostIP:false ConnectorSourceServer:localhost:8080 Provider:aws GoogleProject: DomainFilter:[gexj.club] ExcludeDomains:[] ZoneIDFilter:[] AlibabaCloudConfigFile:/etc/kubernetes/alibaba-cloud.json AlibabaCloudZoneType: AWSZoneType:public AWSZoneTagFilter:[] AWSAssumeRole: AWSBatchChangeSize:1000 AWSBatchChangeInterval:1s AWSEvaluateTargetHealth:true AWSAPIRetries:3 AWSPreferCNAME:false AzureConfigFile:/etc/kubernetes/azure.json AzureResourceGroup: CloudflareProxied:false CloudflareZonesPerPage:50 CoreDNSPrefix:/skydns/ RcodezeroTXTEncrypt:false InfobloxGridHost: InfobloxWapiPort:443 InfobloxWapiUsername:admin InfobloxWapiPassword: InfobloxWapiVersion:2.3.1 InfobloxSSLVerify:true InfobloxView: InfobloxMaxResults:0 DynCustomerName: DynUsername: DynPassword: DynMinTTLSeconds:0 OCIConfigFile:/etc/kubernetes/oci.yaml InMemoryZones:[] PDNSServer:http://localhost:8081 PDNSAPIKey: PDNSTLSEnabled:false TLSCA: TLSClientCert: TLSClientCertKey: Policy:upsert-only Registry:txt TXTOwnerID:/hostedzone/Z3P062U2BQWNDS TXTPrefix: Interval:1m0s Once:false DryRun:false LogFormat:text MetricsAddress::7979 LogLevel:info TXTCacheInterval:0s ExoscaleEndpoint:https://api.exoscale.ch/dns ExoscaleAPIKey: ExoscaleAPISecret: CRDSourceAPIVersion:externaldns.k8s.io/v1alpha1 CRDSourceKind:DNSEndpoint ServiceTypeFilter:[] CFAPIEndpoint: CFUsername: CFPassword: RFC2136Host: RFC2136Port:0 RFC2136Zone: RFC2136Insecure:false RFC2136TSIGKeyName: RFC2136TSIGSecret: RFC2136TSIGSecretAlg: RFC2136TAXFR:false NS1Endpoint: NS1IgnoreSSL:false TransIPAccountName: TransIPPrivateKeyFile:}"
time="2019-09-19T08:06:43Z" level=info msg="Created Kubernetes client <a href="https://10.100.0.1/" "="" >https://10.100.0.1:443"
time="2019-09-19T08:06:47Z" level=info msg="All records are already up to date

apiVersion: v1
kind: ServiceAccount
metadata:
  name: external-dns
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: external-dns
rules:
- apiGroups: [""]
  resources: ["services"]
  verbs: ["get","watch","list"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get","watch","list"]
- apiGroups: ["extensions"]
  resources: ["ingresses"]
  verbs: ["get","watch","list"]
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["list","watch"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: external-dns-viewer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: external-dns
subjects:
- kind: ServiceAccount
  name: external-dns
  namespace: default
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: external-dns
spec:
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: external-dns
    spec:
      serviceAccountName: external-dns
      containers:
      - name: external-dns
        image: registry.opensource.zalan.do/teapot/external-dns:latest
        args:
        - --source=service
        - --source=ingress
        - --domain-filter=gexj.club # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones
        - --provider=aws
        - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization
        - --aws-zone-type=public # only look at public hosted zones (valid values are public, private or no value for both)
        - --registry=txt
        - --txt-owner-id=/hostedzone/Z3P062U2BQWNDS
      securityContext:
        fsGroup: 65534 # For ExternalDNS to be able to read Kubernetes and AWS token files

Verify ExternalDNS works (Service example)

//After roughly two minutes check that a corresponding DNS record for your service was created.

# aws route53 list-resource-record-sets --output json --hosted-zone-id "/hostedzone/Z3P062U2BQWNDS" --query "ResourceRecordSets[?Name == 'nginx.gexj.club.']|[?Type == 'A']"

[
    {
        "Name": "nginx.gexj.club.",
        "Type": "A",
        "AliasTarget": {
            "HostedZoneId": "ZWKZPGTI48KDX",
            "DNSName": "ada69bd16dab411e9b32f061aa86c2a6-1959369398.ap-northeast-2.elb.amazonaws.com.",
            "EvaluateTargetHealth": true
        }
    }
]

apiVersion: v1
kind: Service
metadata:
  name: nginx
  annotations:
    external-dns.alpha.kubernetes.io/hostname: nginx.gexj.club
spec:
  type: LoadBalancer
  ports:
  - port: 80
    name: http
    targetPort: 80
  selector:
    app: nginx
 
---
 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: nginx
spec:
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx
        name: nginx
        ports:
        - containerPort: 80
          name: http