控制节点
kubernetes-server安装
cd /data/work wget https://dl.k8s.io/v1.22.2/kubernetes-server-linux-amd64.tar.gz tar -xf kubernetes-server-linux-amd64.tar.gz cd kubernetes/server/bin/ cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/ rsync -vaz kube-apiserver kube-controller-manager kube-scheduler kubectl master2:/usr/local/bin/ rsync -vaz kube-apiserver kube-controller-manager kube-scheduler kubectl master3:/usr/local/bin/
创建目录
mkdir -p /etc/kubernetes/ # kubernetes组件配置文件存放目录 mkdir -p /etc/kubernetes/ssl # kubernetes组件证书文件存放目录 mkdir /var/log/kubernetes # kubernetes组件日志文件存放目录 mkdir -p /data/work/kubernetes # kubernetes存放证书生成的目录
创建csr请求文件
{ "CN": "kubernetes", "hosts": [ "127.0.0.1", "192.168.32.11", "192.168.32.12", "192.168.32.13", "192.168.32.14", "192.168.32.15", "192.168.32.16", "192.168.32.17", "192.168.32.18", "192.168.32.155", "10.100.0.1", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Hubei", "L": "Wuhan", "O": "k8s", "OU": "system" } ] }
注:10.100.0.1是集群的一个ip
生成证书和token文件
cfssl gencert -ca=etcd/ca.pem -ca-key=etcd/ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kubernetes/kube-apiserver
cat > token.csv << EOF $(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap" EOF
创建配置文件
vim kube-apiserver.conf KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ --anonymous-auth=false \ --bind-address=192.168.32.11 \ --secure-port=6443 \ --advertise-address=192.168.32.11 \ --insecure-port=0 \ --authorization-mode=Node,RBAC \ --runtime-config=api/all=true \ --enable-bootstrap-token-auth \ --service-cluster-ip-range=10.100.0.0/16 \ --token-auth-file=/etc/kubernetes/token.csv \ --service-node-port-range=3000-50000 \ #设置node-port优化 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \ --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \ --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \ --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ # 1.20以上版本必须有此参数 --service-account-issuer=https://kubernetes.default.svc.cluster.local \ # 1.20以上版本必须有此参数 --etcd-cafile=/etc/etcd/ssl/ca.pem \ --etcd-certfile=/etc/etcd/ssl/etcd.pem \ --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \ --etcd-servers=https://192.168.32.11:2379,https://192.168.32.12:2379,https://192.168.32.13:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --apiserver-count=3 \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/log/kube-apiserver-audit.log \ --event-ttl=1h \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=4"
logtostderr:启用日志
v:日志等级
log-dir:日志目录
etcd-servers:etcd集群地址
bind-address:监听地址
secure-port:https安全端口
advertise-address:集群通告地址
allow-privileged:启用授权
service-cluster-ip-range:Service虚拟IP地址段
enable-admission-plugins:准入控制模块
authorization-mode:认证授权,启用RBAC授权和节点自管理
enable-bootstrap-token-auth:启用TLS bootstrap机制
token-auth-file:bootstrap token文件
service-node-port-range:Service nodeport类型默认分配端口范围
kubelet-client-xxx:apiserver访问kubelet客户端证书
tls-xxx-file:apiserver https证书
etcd-xxxfile:连接Etcd集群证书
audit-log-xxx:审计日志
创建服务启动文件
vim /usr/lib/systemd/system/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes After=etcd.service Wants=etcd.service [Service] EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target
同步相关文件到各个节点
cp kubernetes/* /etc/kubernetes/ssl/ cp ca.pem /etc/kubernetes/ssl cp ca-key.pem /etc/kubernetes/ssl cp token.csv /etc/kubernetes/ cp kube-apiserver.conf /etc/kubernetes/ rsync /usr/lib/systemd/system/kube-apiserver.service master2:/usr/lib/systemd/system/kube-apiserver.service rsync /usr/lib/systemd/system/kube-apiserver.service master3:/usr/lib/systemd/system/kube-apiserver.service rsync -vaz token.csv master2:/etc/kubernetes/ rsync -vaz token.csv master3:/etc/kubernetes/ rsync -vaz /etc/kubernetes/ssl/* master2:/etc/kubernetes/ssl/ rsync -vaz /etc/kubernetes/ssl/* master3:/etc/kubernetes/ssl/ rsync -vaz kube-apiserver.conf master2:/etc/kubernetes/ rsync -vaz kube-apiserver.conf master3:/etc/kubernetes/
其他两个节点修改配置文件ip
启动服务
systemctl daemon-reload systemctl enable kube-apiserver systemctl start kube-apiserver systemctl status kube-apiserver curl --insecure https://192.168.32.11:6443/
部署kubectl
创建csr请求文件
{ "CN": "admin", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Hubei", "L": "Wuhan", "O": "system:masters", "OU": "system" } ] }
生成证书
mkdir kubectl cfssl gencert -ca=etcd/ca.pem -ca-key=etcd/ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare kubectl/admin
创建kubeconfig配置文件
kubectl config set-cluster kubernetes --certificate-authority=etcd/ca.pem --embed-certs=true --server=https://192.168.32.155:6443 --kubeconfig=kube.config kubectl config set-credentials admin --client-certificate=kubectl/admin.pem --client-key=kubectl/admin-key.pem --embed-certs=true --kubeconfig=kube.config kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config kubectl config use-context kubernetes --kubeconfig=kube.config mkdir ~/.kube cp kube.config ~/.kube/config kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes
查看集群组件状态
kubectl cluster-info
kubectl get componentstatuses
kubectl get all --all-namespace
同步kubectl配置文件到其他节点
rsync -vaz /root/.kube/config master2:/root/.kube/ rsync -vaz /root/.kube/config master3:/root/.kube/
配置kubectl补全
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
kubectl completion bash > ~/.kube/completion.bash.inc
source '/root/.kube/completion.bash.inc'
source $HOME/.bash_profile
部署kube-controller-manager
创建csr请求文件
vim kube-controller-manager-csr.json { "CN": "system:kube-controller-manager", "key": { "algo": "rsa", "size": 2048 }, "hosts": [ "192.168.32.1", "192.168.32.11", "192.168.32.12", "192.168.32.13" ], "names": [ { "C": "CN", "ST": "Hubei", "L": "Wuhan", "O": "system:kube-controller-manager", "OU": "system" } ] }
生成证书
cfssl gencert -ca=etcd/ca.pem -ca-key=etcd/ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager ls kube-controller-manager*.pem
创建kube-controller-manager的kubeconfig
kubectl config set-cluster kubernetes --certificate-authority=etcd/ca.pem --embed-certs=true --server=https://192.168.32.155:6443 --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
创建配置文件
vim kube-controller-manager.conf KUBE_CONTROLLER_MANAGER_OPTS="--port=0 \ --secure-port=10252 \ --bind-address=127.0.0.1 \ --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \ --service-cluster-ip-range=10.100.0.0/16 \ --cluster-name=kubernetes \ --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \ --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \ --allocate-node-cidrs=true \ --cluster-cidr=10.0.0.0/16 \ --experimental-cluster-signing-duration=87600h \ --root-ca-file=/etc/kubernetes/ssl/ca.pem \ --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \ --leader-elect=true \ --feature-gates=RotateKubeletServerCertificate=true \ --controllers=*,bootstrapsigner,tokencleaner \ --horizontal-pod-autoscaler-use-rest-clients=true \ --horizontal-pod-autoscaler-sync-period=10s \ --tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \ --tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \ --use-service-account-credentials=true \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=2"
创建启动文件
vim kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes [Service] EnvironmentFile=-/etc/kubernetes/kube-controller-manager.conf ExecStart=/usr/local/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target
同步相关文件到各个节点
cp kube-controller-manager*.pem /etc/kubernetes/ssl/ cp kube-controller-manager.kubeconfig /etc/kubernetes/ cp kube-controller-manager.conf /etc/kubernetes/ cp kube-controller-manager.service /usr/lib/systemd/system/ rsync -vaz kube-controller-manager*.pem master2:/etc/kubernetes/ssl/ rsync -vaz kube-controller-manager*.pem master3:/etc/kubernetes/ssl/ rsync -vaz kube-controller-manager.kubeconfig kube-controller-manager.conf master2:/etc/kubernetes/ rsync -vaz kube-controller-manager.kubeconfig kube-controller-manager.conf master3:/etc/kubernetes/ rsync -vaz kube-controller-manager.service master2:/usr/lib/systemd/system/ rsync -vaz kube-controller-manager.service master3:/usr/lib/systemd/system/
启动服务
systemctl daemon-reload systemctl enable kube-controller-manager systemctl start kube-controller-manager systemctl status kube-controller-manager
部署kube-scheduler
创建csr请求文件
vim kube-scheduler-csr.json { "CN": "system:kube-scheduler", "hosts": [ "127.0.0.1", "192.168.32.11", "192.168.32.12", "192.168.32.13" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Hubei", "L": "Wuhan", "O": "system:kube-scheduler", "OU": "system" } ] }
生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler ls kube-scheduler*.pem
创建kube-scheduler的kubeconfig
kubectl config set-cluster kubernetes --certificate-authority=etcd/ca.pem --embed-certs=true --server=https://192.168.32.155:6443 --kubeconfig=kube-scheduler.kubeconfig kubectl config set-credentials system:kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
创建服务启动文件
vim kube-scheduler.service [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/kubernetes/kubernetes [Service] EnvironmentFile=-/etc/kubernetes/kube-scheduler.conf ExecStart=/usr/local/bin/kube-scheduler --leader-elect=true \ /etc/kubernetes/kube-scheduler.kubeconfig Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target
同步相关文件到各个节点
cp kube-scheduler*.pem /etc/kubernetes/ssl/ cp kube-scheduler.kubeconfig /etc/kubernetes/ cp kube-scheduler.conf /etc/kubernetes/ cp kube-scheduler.service /usr/lib/systemd/system/ rsync -vaz kube-scheduler*.pem master2:/etc/kubernetes/ssl/ rsync -vaz kube-scheduler*.pem master3:/etc/kubernetes/ssl/ rsync -vaz kube-scheduler.kubeconfig kube-scheduler.conf master2:/etc/kubernetes/ rsync -vaz kube-scheduler.kubeconfig kube-scheduler.conf master3:/etc/kubernetes/ rsync -vaz kube-scheduler.service master2:/usr/lib/systemd/system/ rsync -vaz kube-scheduler.service master3:/usr/lib/systemd/system/
启动服务
systemctl daemon-reload systemctl enable kube-scheduler systemctl start kube-scheduler systemctl status kube-scheduler
工作节点
部署docker
在三个work节点上安装
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo yum install -y docker-ce systemctl enable docker systemctl start docker docker --version
修改docker源和驱动
cat > /etc/docker/daemon.json << EOF { "exec-opts": ["native.cgroupdriver=systemd"], "registry-mirrors": [ "https://1nj0zren.mirror.aliyuncs.com", "https://kfwkfulq.mirror.aliyuncs.com", "https://2lqq34jg.mirror.aliyuncs.com", "https://pee6w651.mirror.aliyuncs.com", "http://hub-mirror.c.163.com", "https://docker.mirrors.ustc.edu.cn", "http://f1361db2.m.daocloud.io", "https://registry.docker-cn.com" ] } EOF systemctl restart docker docker info | grep "Cgroup Driver"
下载依赖镜像测试
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 k8s.gcr.io/pause:3.2 docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0 docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0 k8s.gcr.io/coredns:1.7.0 docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0
部署kubelet
以下操作在master1上操作
创建kubelet-bootstrap.kubeconfig
BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv) kubectl config set-cluster kubernetes --certificate-authority=etcd/ca.pem --embed-certs=true --server=https://192.168.32.150:6443 --kubeconfig=kubelet-bootstrap.kubeconfig kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
创建配置文件
vim kubelet.json { "kind": "KubeletConfiguration", "apiVersion": "kubelet.config.k8s.io/v1beta1", "authentication": { "x509": { "clientCAFile": "/etc/kubernetes/ssl/ca.pem" }, "webhook": { "enabled": true, "cacheTTL": "2m0s" }, "anonymous": { "enabled": false } }, "authorization": { "mode": "Webhook", "webhook": { "cacheAuthorizedTTL": "5m0s", "cacheUnauthorizedTTL": "30s" } }, "address": "192.168.32.14", "port": 10250, "readOnlyPort": 10255, "cgroupDriver": "cgroupfs", # 如果docker的驱动为systemd,处修改为systemd。此处设置很重要,否则后面node节点无法加入到集群 "hairpinMode": "promiscuous-bridge", "serializeImagePulls": false, "featureGates": { "RotateKubeletClientCertificate": true, "RotateKubeletServerCertificate": true }, "clusterDomain": "cluster.local.", "clusterDNS": ["10.100.0.2"] }
创建启动文件
vim kubelet.service [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/kubernetes/kubernetes After=docker.service Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet ExecStart=/usr/local/bin/kubelet \ --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \ --cert-dir=/etc/kubernetes/ssl \ --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ --config=/etc/kubernetes/kubelet.json \ --network-plugin=cni \ --pod-infra-container-image=k8s.gcr.io/pause:3.2 \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target
–hostname-override:显示名称,集群中唯一
–network-plugin:启用CNI
–kubeconfig:空路径,会自动生成,后面用于连接apiserver
–bootstrap-kubeconfig:首次启动向apiserver申请证书
–config:配置参数文件
–cert-dir:kubelet证书生成目录
–pod-infra-container-image:管理Pod网络容器的镜像
同步相关文件到各个节点
到子节点
cp kubelet-bootstrap.kubeconfig /etc/kubernetes/ cp kubelet.json /etc/kubernetes/ cp kubelet.service /usr/lib/systemd/system/ for i in node1 node2 node3;do rsync -vaz kubelet-bootstrap.kubeconfig kubelet.json $i:/etc/kubernetes/;done for i in node1 node2 node3;do rsync -vaz ca.pem $i:/etc/kubernetes/ssl/;done for i in node1 node2 node3;do rsync -vaz kubelet.service $i:/usr/lib/systemd/system/;done
kubelete.json配置文件address改为各个节点的ip地址
启动服务
mkdir /var/lib/kubelet mkdir /var/log/kubernetes systemctl daemon-reload systemctl enable kubelet systemctl start kubelet systemctl status kubelet
查看节点 csr
kubectl certificate approve node-csr-HlX3cExsZohWsu8Dd6Rp_ztFejmMdpzvti_qgxo4SAQ kubectl certificate approve node-csr-oykYfnH_coRF2PLJH4fOHlGznOZUBPDg5BPZXDo2wgk kubectl certificate approve node-csr-ytRB2fikhL6dykcekGg4BdD87o-zw9WPU44SZ1nFT50 kubectl get csr kubectl get nodes
部署kube-proxy
vim kube-proxy-csr.json { "CN": "system:kube-proxy", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Hubei", "L": "Wuhan", "O": "k8s", "OU": "system" } ] }
生成证书
cfssl gencert -ca=etcd/ca.pem -ca-key=etcd/ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
创建kubeconfig文件
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://172.10.0.20:6443 --kubeconfig=kube-proxy.kubeconfig kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
创建kube-proxy配置文件
vim kube-proxy.yaml apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 192.168.32.14 clientConnection: kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig clusterCIDR: 10.10.0.0/16 # 此处网段必须与网络组件网段保持一致,否则部署网络组件时会报错 healthzBindAddress: 192.168.32.14:10256 kind: KubeProxyConfiguration metricsBindAddress: 192.168.32.14:10249 mode: "ipvs"
创建服务启动文件
vim kube-proxy.service [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/kubernetes/kubernetes After=network.target [Service] WorkingDirectory=/var/lib/kube-proxy ExecStart=/usr/local/bin/kube-proxy \ --config=/etc/kubernetes/kube-proxy.yaml \ --alsologtostderr=true \ --logtostderr=false \ --log-dir=/var/log/kubernetes \ --v=2 Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target
同步文件到各个节点
cp kube-proxy*.pem /etc/kubernetes/ssl/ cp kube-proxy.kubeconfig kube-proxy.yaml /etc/kubernetes/ cp kube-proxy.service /usr/lib/systemd/system/ for i in node1 node2 node3;do rsync -vaz kube-proxy.kubeconfig kube-proxy.yaml $i:/etc/kubernetes/;done for i in node1 node2 node3;do rsync -vaz kube-proxy.service $i:/usr/lib/systemd/system/;done
配置文件kube-proxy.yaml中address修改为各节点的实际IP
启动服务
mkdir -p /var/lib/kube-proxy systemctl daemon-reload systemctl enable kube-proxy systemctl restart kube-proxy systemctl status kube-proxy
配置网络组件
wget https://docs.projectcalico.org/v3.14/manifests/calico.yaml kubectl apply -f calico.yaml
查看节点
kubectl get pods -A
部署coredns
部署时 如果报错 修改宿主机的配置/etc/hosts #127.0.0.1:53
下载coredns yaml文件
https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
修改yaml文件
kubernetes cluster.local in-addr.arpa ip6.arpa forward . /etc/resolv.conf clusterIP为:10.100.0.2
安装coredns
kubectl apply -f coredns.yaml
至此 二进制安装k8s就到此结束~