k8s 部署手册 - v04
时间:2023-02-27 16:18 来源:未知 作者:IT
一、基础配置
1. 修改主机名
1.hostnamectl set-hostname k8s-master01
2.hostnamectl set-hostname k8s-master02
3.hostnamectl set-hostname k8s-master03
4.hostnamectl set-hostname k8s-node01
5.hostnamectl set-hostname k8s-node02
2. 添加 主机名与 IP 地址解析
cat > /etc/hosts <<EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.62 lookup apiserver.cluster.local
192.168.1.60 k8s-master01
192.168.1.61 k8s-master02
192.168.1.62 k8s-master03
192.168.1.63 k8s-node01
192.168.1.64 k8s-node02
EOF
3. 升级服务器内核,时间同步,关闭防火墙,重启服务器
#添加访问互联路由
cat > /etc/resolv.conf <<EOF
nameserver 114.114.114.114
nameserver 8.8.8.8
EOF
cat /etc/resolv.conf
# ssh连接Linux比较慢
#sed -i "s|#UseDNS yes|UseDNS no|" /etc/ssh/sshd_config
#sed -i "s|GSSAPIAuthentication yes|GSSAPIAuthentication no|" /etc/ssh/sshd_config
#设置为阿里云yum源
rm -rf /etc/yum.repos.d/bak && mkdir -p /etc/yum.repos.d/bak && mv /etc/yum.repos.d/* /etc/yum.repos.d/bak
curl -o /etc/yum.repos.d/CentOS-7.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all && yum makecache
cd /etc/yum.repos.d
#CentOS7使用/etc/rc.d/rc.local设置开机自动启动
chmod +x /etc/rc.d/rc.local
#安装依赖包
yum -y install vim net-tools lrzsz unzip gcc telnet wget sshpass ntpdate ntp curl
yum -y install conntrack ipvsadm ipset jq iptables sysstat libseccomp git
#时间同步
echo '*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com >/dev/null 2>&1'>/var/spool/cron/root && crontab -l
#设置防火墙为 Iptables 并设置空规则
systemctl stop firewalld && systemctl disable firewalld
yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save
#关闭 SELINUX
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
#调整内核参数,对于 K8S
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
#net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间,只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
modprobe ip_vs_rr && modprobe br_netfilter && sysctl -p /etc/sysctl.d/kubernetes.conf
#关闭系统不需要服务
systemctl stop postfix && systemctl disable postfix
4. 升级内核,重启服务器
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum -y install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
yum --enablerepo="elrepo-kernel" -y install kernel-lt.x86_64
awk -F \' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
grub2-set-default "CentOS Linux (5.4.225-1.el7.elrepo.x86_64) 7 (Core)"
#grub2-set-default 'CentOS Linux (4.4.222-1.el7.elrepo.x86_64) 7 (Core)'
#重启服务器
reboot
################################
二、sealos 部署 k8s-v1.19
1. 安装 sealos3.3
#添加访问互联路由
cat > /etc/resolv.conf <<EOF
nameserver 8.8.8.8
nameserver 114.114.114.114
nameserver 223.5.5.5
EOF
cat /etc/resolv.conf
#时间同步
ntpdate ntp1.aliyun.com
wget -c https://github.com/fanux/sealos/releases/download/v3.3.8/sealos
tar zxvf sealos*.tar.gz sealos && chmod +x sealos && mv sealos /usr/bin
sealos version
#时间同步
ntpdate ntp1.aliyun.com
2. 离线安装 k8s 1.19
链接:https://pan.baidu.com/s/1F9sZoHBX1K1ihBP9rZSHBQ?pwd=jood
提取码:jood
#安装
sealos init --passwd 1qaz@WSX \
--master 192.168.1.60 \
--master 192.168.1.61 \
--master 192.168.1.62 \
--node 192.168.1.63 \
--node 192.168.1.64 \
--pkg-url /root/kube1.19.16.tar.gz \
--version v1.19.16
3. 验证集群
kubectl get nodes
kubectl get pod -A
#配置kubectl自动补全
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> /etc/profile
#查看污点
kubectl describe node |grep -i taints
#去除污点
#kubectl taint node k8s-master02 node-role.kubernetes.io/master:NoSchedule-
#kubectl taint node k8s-master03 node-role.kubernetes.io/master:NoSchedule-
4.sealos3.3 常用命令
#添加 node 节点:
sealos join --node 192.168.1.63,192.168.1.64
#添加master
sealos join -master 192.168.1.61,192.168.1.62
#删除 node 节点:
sealos clean --node 192.168.1.63,192.168.1.64
#删除 master 节点:
sealos clean --master 192.168.1.61,192.168.1.62
#重置集群
sealos clean --all -f
5. 安装 top 命令
cat > /root/top.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --kubelet-insecure-tls
- --secure-port=4443
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
#这里可以自己把metrics-server做到自己的阿里云镜像里面,并把下面替换成自己的镜像地址
image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/metrics-server:v0.4.3
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 10
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 100
EOF
kubectl apply -f /root/top.yaml
三、部署 nfs
1. 服务端
#添加访问互联路由
cat > /etc/resolv.conf <<EOF
nameserver 114.114.114.114
nameserver 8.8.8.8
EOF
# 我们这里在192.168.1.60上安装(在生产中,大家要提供作好NFS-SERVER环境的规划)
yum -y install nfs-utils
# 创建NFS挂载目录
mkdir /nfs_dir
chown nobody.nobody /nfs_dir
# 修改NFS-SERVER配置
echo '/nfs_dir *(rw,sync,no_root_squash)' > /etc/exports
# 重启服务
systemctl restart rpcbind.service
systemctl restart nfs-utils.service
systemctl restart nfs-server.service
# 增加NFS-SERVER开机自启动
systemctl enable rpcbind.service
systemctl enable nfs-utils.service
systemctl enable nfs-server.service
# 验证NFS-SERVER是否能正常访问
#showmount -e 192.168.1.60
2. 客户端
#需要挂载的服务器执行
mkdir /nfs_dir
yum install nfs-utils -y
#挂载
mount 192.168.1.60:/nfs_dir /nfs_dir
#添加开机挂载
echo "mount 192.168.1.60:/nfs_dir /nfs_dir" >> /etc/rc.local
cat /etc/rc.local
四、部署 StorageClass
1. 创建 nfs-sc.yaml
cat > /root/nfs-sc.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: kube-system
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-provisioner-01
namespace: kube-system
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-provisioner-01
template:
metadata:
labels:
app: nfs-provisioner-01
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
#老版本插件使用jmgao1983/nfs-client-provisioner:latest
# image: jmgao1983/nfs-client-provisioner:latest
image: vbouchaud/nfs-client-provisioner:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: nfs-provisioner-01 # 此处供应者名字供storageclass调用
- name: NFS_SERVER
value: 192.168.1.60 # 填入NFS的地址
- name: NFS_PATH
value: /nfs_dir # 填入NFS挂载的目录
volumes:
- name: nfs-client-root
nfs:
server: 192.168.1.60 # 填入NFS的地址
path: /nfs_dir # 填入NFS挂载的目录
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs-boge
provisioner: nfs-provisioner-01
# Supported policies: Delete、 Retain , default is Delete
reclaimPolicy: Retain
EOF
#创建
kubectl apply -f /root/nfs-sc.yaml
#查看
kubectl -n kube-system get pod
kubectl get sc
五、harbor 仓库搭建
1. 安装
#目录/root上传文件docker-compose和harbor-offline-installer-v1.2.0.tgz
mv /root/docker-compose /usr/local/bin/
chmod a+x /usr/local/bin/docker-compose
ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
tar -zxvf harbor-offline-installer-v2.4.1.tgz
mv harbor /usr/local/
cd /usr/local/harbor/
cp harbor.yml.tmpl harbor.yml
sed -i 's/hostname: reg.mydomain.com/hostname: 192.168.1.77/g' harbor.yml
sed -i 's/https/#https/g' harbor.yml
sed -i 's/certificate/#certificate/g' harbor.yml
sed -i 's/private_key/#private_key/g' harbor.yml
#数据库目录
mkdir /data
cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://nr240upq.mirror.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn", "http://hub-mirror.c.163.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"insecure-registries": ["192.168.1.77:80"]
}
systemctl daemon-reload && systemctl restart docker
#安装
./install.sh
## 重启harbor
cd /usr/local/harbor/
docker-compose down -v
docker-compose up -d
docker ps|grep harbor
netstat -ntlp
2. 需要访问仓库的其他节点的 daemon.json 添加如下内容
##-------------------
vim /etc/docker/daemon.json
"registry-mirrors": ["https://nr240upq.mirror.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn"],
"insecure-registries": ["192.168.1.77:80"],
##-------------------
#重启
systemctl daemon-reload && systemctl restart docker
3. 节点使用仓库
#登入仓库网站
docker login -u admin -p Harbor12345 192.168.1.77:80
#下载镜像
docker pull daocloud.io/library/nginx:1.9.1
#给镜像打上标签
docker tag daocloud.io/library/nginx:1.9.1 192.168.1.77:80/library/nginx:1.9.1
#镜像上传
docker push 192.168.1.77:80/library/nginx:1.9.1
#删除镜像
docker rmi 192.168.1.77:80/library/nginx:1.9.1
#将镜像保存为本地tar文件,
docker save k8s.gcr.io/coredns:1.7.0 > /root/coredns-v1.7.0.tar
#使用load加载tar文件
docker load -i /root/coredns-v1.7.0.tar
4. 批量打包上传 harbor 镜像
cd /root
#查看服务器镜像名称
docker images | awk 'NR!=1{print $1":"$2}' > 01-image-old.txt && cat 01-image-old.txt
# /换成-
rm -rf 02-image-sed.txt && cp 01-image-old.txt 02-image-sed.txt && sed -i "s|/|-|g" 02-image-sed.txt && cat /root/02-image-sed.txt
#打标签harbor仓库
vim /root/03-tar-image.sh
#####################################################
#!/bin/sh
old=/root/01-image-old.txt
new=/root/02-image-sed.txt
l=$(cat /root/01-image-old.txt| wc -l)
for ((i=1 ; i<=$l ; i++))
do
a=$(sed -n "$i"p $old)
b=$(sed -n "$i"p $new)
#echo "update xxxx set uid='$a' where uid='$b';"
docker tag $a 192.168.1.77:80/library/$b
done
#####################################################
#运行打仓库标签
bash /root/03-tar-image.sh
docker images |grep library
#查看打标harbor仓库images名称
docker images |grep 192.168.1.77 | awk '{print $1":"$2}' > 04-tar-image.txt && cat 04-tar-image.txt
#上传到harbor仓库
for h in `cat 04-tar-image.txt`; do docker push $h; done
#删除打标镜像
for d in `cat 04-tar-image.txt`; do docker rmi $d; done
docker images |grep library
#删除创建的文件
rm -rf /root/0*txt 03-tar-image.sh
六、kuboard 界面管理
1. 下载地址
curl -o kuboard-v3.yaml https://addons.kuboard.cn/kuboard/kuboard-v3-storage-class.yaml
2. 编辑 yaml
#编辑 kuboard-v3.yaml 文件中的配置,该部署文件中,有1处配置必须修改:storageClassName
volumeClaimTemplates:
- metadata:
name: data
spec:
# 请填写一个有效的 StorageClass name
storageClassName: nfs-boge
accessModes: [ "ReadWriteMany" ]
resources:
requests:
storage: 5Gi
3. 执行
kubectl create -f kuboard-v3.yaml
kubectl get pod -n kuboard
############################################
#访问
http://192.168.1.60:30080/
输入初始用户名和密码,并登录
用户名: admin
密码: Kuboard123
#############################################
#查看错误
journalctl -f -u kubelet.service
七、helm3 安装
1.helm 包下载地址
wget https://get.helm.sh/helm-v3.6.1-linux-amd64.tar.gz
2. 安装 helm
#解压 && 移动到 /usr/bin 目录下:
tar -xvf helm-v3.6.1-linux-amd64.tar.gz && cd linux-amd64/ && mv helm /usr/bin
#查看版本
helm version
3. 配置仓库
#添加公用的仓库
helm repo add incubator https://charts.helm.sh/incubator
helm repo add bitnami https://charts.bitnami.com/bitnami
# 配置helm微软源地址
helm repo add stable http://mirror.azure.cn/kubernetes/charts
# 配置helm阿里源地址
helm repo add aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
helm repo add google https://kubernetes-charts.storage.googleapis.com
helm repo add jetstack https://charts.jetstack.io
# 查看仓库
helm repo list
# 更新仓库
helm repo update
# 删除仓库
#helm repo remove aliyun
# helm list
八、haproxy+keepalived+ingress
1. 部署阿里云 ingress
mkdir -p /data/k8s/
cd /data/k8s/
cat > /data/k8s/aliyun-ingress-nginx.yaml <<EOF
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
app: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-controller
labels:
app: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
- "ingress-controller-leader-nginx"
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-controller
labels:
app: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-controller
subjects:
- kind: ServiceAccount
name: nginx-ingress-controller
namespace: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
labels:
app: ingress-nginx
name: nginx-ingress-lb
namespace: ingress-nginx
spec:
# DaemonSet need:
# ----------------
type: ClusterIP
# ----------------
# Deployment need:
# ----------------
# type: NodePort
# ----------------
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
- name: https
port: 443
targetPort: 443
protocol: TCP
- name: metrics
port: 10254
protocol: TCP
targetPort: 10254
selector:
app: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app: ingress-nginx
data:
keep-alive: "75"
keep-alive-requests: "100"
upstream-keepalive-connections: "10000"
upstream-keepalive-requests: "100"
upstream-keepalive-timeout: "60"
allow-backend-server-header: "true"
enable-underscores-in-headers: "true"
generate-request-id: "true"
http-redirect-code: "301"
ignore-invalid-headers: "true"
log-format-upstream: '{"@timestamp": "$time_iso8601","remote_addr": "$remote_addr","x-forward-for": "$proxy_add_x_forwarded_for","request_id": "$req_id","remote_user": "$remote_user","bytes_sent": $bytes_sent,"request_time": $request_time,"status": $status,"vhost": "$host","request_proto": "$server_protocol","path": "$uri","request_query": "$args","request_length": $request_length,"duration": $request_time,"method": "$request_method","http_referrer": "$http_referer","http_user_agent": "$http_user_agent","upstream-sever":"$proxy_upstream_name","proxy_alternative_upstream_name":"$proxy_alternative_upstream_name","upstream_addr":"$upstream_addr","upstream_response_length":$upstream_response_length,"upstream_response_time":$upstream_response_time,"upstream_status":$upstream_status}'
max-worker-connections: "65536"
worker-processes: "2"
proxy-body-size: 20m
proxy-connect-timeout: "10"
proxy_next_upstream: error timeout http_502
reuse-port: "true"
server-tokens: "false"
ssl-ciphers: ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl-protocols: TLSv1 TLSv1.1 TLSv1.2
ssl-redirect: "false"
worker-cpu-affinity: auto
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app: ingress-nginx
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app: ingress-nginx
annotations:
component.version: "v0.30.0"
component.revision: "v1"
spec:
# Deployment need:
# ----------------
# replicas: 1
# ----------------
selector:
matchLabels:
app: ingress-nginx
template:
metadata:
labels:
app: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
scheduler.alpha.kubernetes.io/critical-pod: ""
spec:
# DaemonSet need:
# ----------------
hostNetwork: true
# ----------------
serviceAccountName: nginx-ingress-controller
priorityClassName: system-node-critical
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- ingress-nginx
topologyKey: kubernetes.io/hostname
weight: 100
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: type
operator: NotIn
values:
- virtual-kubelet
containers:
- name: nginx-ingress-controller
image: registry.cn-beijing.aliyuncs.com/acs/aliyun-ingress-controller:v0.30.0.2-9597b3685-aliyun
args:
- /nginx-ingress-controller
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb
- --annotations-prefix=nginx.ingress.kubernetes.io
- --enable-dynamic-certificates=true
- --v=2
securityContext:
allowPrivilegeEscalation: true
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
runAsUser: 101
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
# resources:
# limits:
# cpu: "1"
# memory: 2Gi
# requests:
# cpu: "1"
# memory: 2Gi
volumeMounts:
- mountPath: /etc/localtime
name: localtime
readOnly: true
volumes:
- name: localtime
hostPath:
path: /etc/localtime
type: File
nodeSelector:
boge/ingress-controller-ready: "true"
tolerations:
- operator: Exists
initContainers:
- command:
- /bin/sh
- -c
- |
mount -o remount rw /proc/sys
sysctl -w net.core.somaxconn=65535
sysctl -w net.ipv4.ip_local_port_range="1024 65535"
sysctl -w fs.file-max=1048576
sysctl -w fs.inotify.max_user_instances=16384
sysctl -w fs.inotify.max_user_watches=524288
sysctl -w fs.inotify.max_queued_events=16384
image: registry.cn-beijing.aliyuncs.com/acs/busybox:v1.29.2
imagePullPolicy: Always
name: init-sysctl
securityContext:
privileged: true
procMount: Default
---
## Deployment need for aliyun'k8s:
#apiVersion: v1
#kind: Service
#metadata:
# annotations:
# service.beta.kubernetes.io/alibaba-cloud-loadbalancer-id: "lb-xxxxxxxxxxxxxxxxxxx"
# service.beta.kubernetes.io/alibaba-cloud-loadbalancer-force-override-listeners: "true"
# labels:
# app: nginx-ingress-lb
# name: nginx-ingress-lb-local
# namespace: ingress-nginx
#spec:
# externalTrafficPolicy: Local
# ports:
# - name: http
# port: 80
# protocol: TCP
# targetPort: 80
# - name: https
# port: 443
# protocol: TCP
# targetPort: 443
# selector:
# app: ingress-nginx
# type: LoadBalancer
EOF
kubectl apply -f /data/k8s/aliyun-ingress-nginx.yaml
2. 节点打标签
#允许节点打标签
kubectl label node k8s-master01 boge/ingress-controller-ready=true
kubectl label node k8s-master02 boge/ingress-controller-ready=true
kubectl label node k8s-master03 boge/ingress-controller-ready=true
#删除标签
#kubectl label node k8s-master01 boge/ingress-controller-ready=true --overwrite
#kubectl label node k8s-master02 boge/ingress-controller-ready=true --overwrite
#kubectl label node k8s-master03 boge/ingress-controller-ready=true --overwrite
3.haproxy+keepalived 部署
3.0 部署
yum install haproxy keepalived -y
#重启程序
systemctl restart haproxy.service
systemctl restart keepalived.service
# 查看运行状态
systemctl status haproxy.service
systemctl status keepalived.service
#开机自启动
systemctl enable keepalived.service
systemctl enable haproxy.service
3.1 修改配置 haproxy
vim /etc/haproxy/haproxy.cfg
###################################################
listen ingress-http
bind 0.0.0.0:80
mode tcp
option tcplog
option dontlognull
option dontlog-normal
balance roundrobin
server 192.168.1.60 192.168.1.60:80 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.61 192.168.1.61:80 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.62 192.168.1.62:80 check inter 2000 fall 2 rise 2 weight 1
listen ingress-https
bind 0.0.0.0:443
mode tcp
option tcplog
option dontlognull
option dontlog-normal
balance roundrobin
server 192.168.1.60 192.168.1.60:443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.61 192.168.1.61:443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.62 192.168.1.62:443 check inter 2000 fall 2 rise 2 weight 1
3.2 A 机器修改 keepalived 配置
cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
router_id lb-master
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -60
}
vrrp_instance VI-kube-master {
state MASTER
priority 120
unicast_src_ip 192.168.1.63 #本机ip
unicast_peer {
192.168.1.64 #另一台机器ip
}
dont_track_primary
interface ens33 # 注意这里的网卡名称修改成你机器真实的内网网卡名称,可用命令ip addr查看
virtual_router_id 111
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.1.100 #vip 地址
}
}
EOF
3.3 B 机器修改 keepalived 配置
cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
router_id lb-master
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -60
}
vrrp_instance VI-kube-master {
state MASTER
priority 120
unicast_src_ip 192.168.1.64 #本机ip
unicast_peer {
192.168.1.63 #另一台机器ip
}
dont_track_primary
interface ens33 # 注意这里的网卡名称修改成你机器真实的内网网卡名称,可用命令ip addr查看
virtual_router_id 111
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.1.100 #vip 地址
}
}
EOF
3.4 重启
#重启程序
systemctl restart haproxy.service
systemctl restart keepalived.service
# 查看运行状态
systemctl status haproxy.service
systemctl status keepalived.service
4. 部署 nginx-ingress
cat > /root/nginx-ingress.yaml <<EOF
apiVersion: v1
kind: Service
metadata:
namespace: test
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: nginx
---
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: test
name: nginx
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
namespace: test
name: nginx-ingress
spec:
rules:
- host: nginx.boge.com
http:
paths:
- backend:
serviceName: nginx
servicePort: 80
path: /
EOF
5. 测试 nginx-ingress
kubectl apply -f /root/nginx-ingress.yaml
#查看创建的ingress资源
kubectl get ingress -A
#服务器新增域名解析
echo "192.168.1.100 nginx.boge.com" >> /etc/hosts
# 我们在其它节点上,加下本地hosts,来测试下效果
20.6.1.226 nginx.boge.com
#测试
curl nginx.boge.com
九、elk 日志监控
1. 创建测试 tomcat
cat > 01-tomcat-test.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: tomcat
name: tomcat
spec:
replicas: 1
selector:
matchLabels:
app: tomcat
template:
metadata:
labels:
app: tomcat
spec:
tolerations:
- key: "node-role.kubernetes.io/master"
effect: "NoSchedule"
containers:
- name: tomcat
image: "tomcat:7.0"
env: # 注意点一,添加相应的环境变量(下面收集了两块日志1、stdout 2、/usr/local/tomcat/logs/catalina.*.log)
- name: aliyun_logs_tomcat-syslog # 如日志发送到es,那index名称为 tomcat-syslog
value: "stdout"
- name: aliyun_logs_tomcat-access # 如日志发送到es,那index名称为 tomcat-access
value: "/usr/local/tomcat/logs/catalina.*.log"
volumeMounts: # 注意点二,对pod内要收集的业务日志目录需要进行共享,可以收集多个目录下的日志文件
- name: tomcat-log
mountPath: /usr/local/tomcat/logs
volumes:
- name: tomcat-log
emptyDir: {}
EOF
kubectl apply -f 01-tomcat-test.yaml
2. 部署 elasticsearch
cat > 02-elasticsearch.6.8.13-statefulset.yaml <<EOF
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
addonmanager.kubernetes.io/mode: Reconcile
k8s-app: elasticsearch-logging
version: v6.8.13
name: elasticsearch-logging
namespace: logging
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: elasticsearch-logging
version: v6.8.13
serviceName: elasticsearch-logging
template:
metadata:
labels:
k8s-app: elasticsearch-logging
version: v6.8.13
spec:
# nodeSelector:
# esnode: "true" ## 注意给想要运行到的node打上相应labels
containers:
- env:
- name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: cluster.name
value: elasticsearch-logging-0
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
image: elastic/elasticsearch:6.8.13
name: elasticsearch-logging
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
volumeMounts:
- mountPath: /usr/share/elasticsearch/data
name: elasticsearch-logging
dnsConfig:
options:
- name: single-request-reopen
initContainers:
- command:
- /bin/sysctl
- -w
- vm.max_map_count=262144
image: busybox
imagePullPolicy: IfNotPresent
name: elasticsearch-logging-init
resources: {}
securityContext:
privileged: true
- name: fix-permissions
image: busybox
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: elasticsearch-logging
mountPath: /usr/share/elasticsearch/data
volumes:
- name: elasticsearch-logging
hostPath:
path: /esdata
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: elasticsearch-logging
name: elasticsearch
namespace: logging
spec:
ports:
- port: 9200
protocol: TCP
targetPort: db
selector:
k8s-app: elasticsearch-logging
type: ClusterIP
kubectl apply -f 02-elasticsearch.6.8.13-statefulset.yaml
3. 部署 kibana
cat > 03-kibana.6.8.13.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: logging
labels:
app: kibana
spec:
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: elastic/kibana:6.8.13
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_URL
value: http://elasticsearch:9200
ports:
- containerPort: 5601
---
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: logging
labels:
app: kibana
spec:
ports:
- port: 5601
protocol: TCP
targetPort: 5601
type: ClusterIP
selector:
app: kibana
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kibana
namespace: logging
spec:
rules:
- host: kibana.boge.com
http:
paths:
- path: /
backend:
serviceName: kibana
servicePort: 5601
kubectl apply -f 03-kibana.6.8.13.yaml
4. 部署 log-pilot
cat > 04-log-pilot.yml <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: log-pilot
namespace: logging
labels:
app: log-pilot
# 设置期望部署的namespace
spec:
selector:
matchLabels:
app: log-pilot
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: log-pilot
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
# 是否允许部署到Master节点上
#tolerations:
#- key: node-role.kubernetes.io/master
# effect: NoSchedule
containers:
- name: log-pilot
# 版本请参考https://github.com/AliyunContainerService/log-pilot/releases
image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
resources:
limits:
memory: 500Mi
requests:
cpu: 200m
memory: 200Mi
env:
- name: "NODE_NAME"
valueFrom:
fieldRef:
fieldPath: spec.nodeName
##--------------------------------
# - name: "LOGGING_OUTPUT"
# value: "logstash"
# - name: "LOGSTASH_HOST"
# value: "logstash-g1"
# - name: "LOGSTASH_PORT"
# value: "5044"
##--------------------------------
- name: "LOGGING_OUTPUT"
value: "elasticsearch"
## 请确保集群到ES网络可达
- name: "ELASTICSEARCH_HOSTS"
value: "elasticsearch:9200"
## 配置ES访问权限
#- name: "ELASTICSEARCH_USER"
# value: "{es_username}"
#- name: "ELASTICSEARCH_PASSWORD"
# value: "{es_password}"
##--------------------------------
## https://github.com/AliyunContainerService/log-pilot/blob/master/docs/filebeat/docs.md
## to file need configure 1
# - name: LOGGING_OUTPUT
# value: file
# - name: FILE_PATH
# value: /tmp
# - name: FILE_NAME
# value: filebeat.log
volumeMounts:
- name: sock
mountPath: /var/run/docker.sock
- name: root
mountPath: /host
readOnly: true
- name: varlib
mountPath: /var/lib/filebeat
- name: varlog
mountPath: /var/log/filebeat
- name: localtime
mountPath: /etc/localtime
readOnly: true
## to file need configure 2
# - mountPath: /tmp
# name: mylog
livenessProbe:
failureThreshold: 3
exec:
command:
- /pilot/healthz
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 2
securityContext:
capabilities:
add:
- SYS_ADMIN
terminationGracePeriodSeconds: 30
volumes:
- name: sock
hostPath:
path: /var/run/docker.sock
- name: root
hostPath:
path: /
- name: varlib
hostPath:
path: /var/lib/filebeat
type: DirectoryOrCreate
- name: varlog
hostPath:
path: /var/log/filebeat
type: DirectoryOrCreate
- name: localtime
hostPath:
path: /etc/localtime
## to file need configure 3
# - hostPath:
# path: /tmp/mylog
# type: ""
# name: mylog
kubectl apply -f 04-log-pilot.yml
5. 配置 kibana 页面
Managenment>index Patterns>Create index pattern
#创建日志
Create index pattern> index pattern(tomcat-access*)>Next step
#创建时间
Time Filter field name(@timestamp)>Create index pattern
#查看日志展示
Discover>tomcat-access*
————————————————
版权声明:本文为CSDN博主「大虾别跑」的原创文章,遵循CC 4.0 BY-SA版权协议,转载请附上原文出处链接及本声明。
原文链接:https://blog.csdn.net/qq_35583325/article/details/128172276
十、Prometheus 监控
1. 导入离线包
链接:https://pan.baidu.com/s/1DyMJPT8r_TUpI8Dr31SVew?pwd=m1bk
提取码:m1bk
#导入上传tar包
sudo docker load -i alertmanager-v0.21.0.tar
sudo docker load -i grafana-7.3.4.tar
sudo docker load -i k8s-prometheus-adapter-v0.8.2.tar
sudo docker load -i kube-rbac-proxy-v0.8.0.tar
sudo docker load -i kube-state-metrics-v1.9.7.tar
sudo docker load -i node-exporter-v1.0.1.tar
sudo docker load -i prometheus-config-reloader-v0.43.2.tar
sudo docker load -i prometheus_demo_service.tar
sudo docker load -i prometheus-operator-v0.43.2.tar
sudo docker load -i prometheus-v2.22.1.tar
2. 主节点创建
#解压下载的代码包
sudo unzip kube-prometheus-master.zip
sudo rm -f kube-prometheus-master.zip && cd kube-prometheus-master
#这里建议先看下有哪些镜像,便于在下载镜像快的节点上先收集好所有需要的离线docker镜像
find ./ -type f |xargs grep 'image: '|sort|uniq|awk '{print $3}'|grep ^[a-zA-Z]|grep -Evw 'error|kubeRbacProxy'|sort -rn|uniq
kubectl create -f manifests/setup
kubectl create -f manifests/
#过一会查看创建结果:
kubectl -n monitoring get all
# 附:清空上面部署的prometheus所有服务:
# kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
3. 访问下 prometheus 的 UI
# 修改下prometheus UI的service模式,便于我们访问
# kubectl -n monitoring patch svc prometheus-k8s -p '{"spec":{"type":"NodePort"}}'
service/prometheus-k8s patched
# kubectl -n monitoring get svc prometheus-k8s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
prometheus-k8s NodePort 10.68.23.79 <none> 9090:22129/TCP 7m43s
3.1 修改用户权限
# kubectl edit clusterrole prometheus-k8s
#------ 原始的rules -------
rules:
- apiGroups:
- ""
resources:
- nodes/metrics
verbs:
- get
- nonResourceURLs:
- /metrics
verbs:
- get
#---------------------------
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus-k8s
rules:
- apiGroups:
- ""
resources:
- nodes
- services
- endpoints
- pods
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
- nodes/metrics
verbs:
- get
- nonResourceURLs:
- /metrics
verbs:
- get
4. 监控 ingress-nginx
cat > servicemonitor.yaml <<EOF
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app: ingress-nginx
name: nginx-ingress-scraping
namespace: ingress-nginx
spec:
endpoints:
- interval: 30s
path: /metrics
port: metrics
jobLabel: app
namespaceSelector:
matchNames:
- ingress-nginx
selector:
matchLabels:
app: ingress-nginx
EOF
kubectl apply -f servicemonitor.yaml
kubectl -n ingress-nginx get servicemonitors.monitoring.coreos.com
十一、安装 kubesphere3.3
官网参考文档
https://kubesphere.com.cn/docs/v3.3/pluggable-components/alerting/
1. 部署 kubesphere 时需要默认 StorageClass
kubectl edit sc nfs-boge
metadata:
annotations:
storageclass.beta.kubernetes.io/is-default-class: "true"
2. 下载 yaml
wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/kubesphere-installer.yaml
wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/cluster-configuration.yaml
#修改cluster-configuration.yaml
#将ectd下的 endpointIps改为你的master节点的私有IP地址。
#endpointIps: XX.X.X.X
3. 运行 yaml
kubectl apply -f kubesphere-installer.yaml
kubectl apply -f cluster-configuration.yaml
4. 查看日志
kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f
#访问任意机器的 30880端口
#账号 : admin
#密码 : P@88w0rd
5. 解决 etcd 监控证书找不到问题
kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs \
--from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt \
--from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
--from-file=etcd-client.key=/etc/kubernetes/pki/etcd/healthcheck-client.key
6. 在安装后启用告警系统
在 cluster-configuration.yaml 文件中,搜索 alerting,将 enabled 的 false 更改为 true 以启用告警系统。完成后保存文件
alerting:
enabled: true # 将“false”更改为“true”。
#运行
kubectl apply -f kubesphere-installer.yaml
kubectl apply -f cluster-configuration.yaml
6.0 配置钉钉报警
6.1 钉钉自定义机器配置
添加自定义机器人,安全配置,勾选 ** 加签 **
6.2 操作步骤
左上角 > 平台管理 > 平台设置 > 通知管理 > 通知配置 > 钉钉 > 群机器人配置
开启 - 已启用
填写自己的 Webhook URL
填写自己的 密钥 (加签)
发送测试信息
确定
查看钉钉群消息。是否发送成功?????
7. 在安装后启用应用商店,
在该 YAML 文件中,搜索 openpitrix,将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置。
openpitrix:
store:
enabled: true # 将“false”更改为“true”。
#运行
kubectl apply -f kubesphere-installer.yaml
kubectl apply -f cluster-configuration.yaml
8. 在安装后启用服务网格 istio
在该配置文件中,搜索 servicemesh,并将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置
servicemesh:
enabled: true # 将“false”更改为“true”。
istio: # Customizing the istio installation configuration, refer to https://istio.io/latest/docs/setup/additional-setup/customize-installation/
components:
ingressGateways:
- name: istio-ingressgateway # 将服务暴露至服务网格之外。默认不开启。
enabled: false
cni:
enabled: false # 启用后,会在 Kubernetes pod 生命周期的网络设置阶段完成 Istio 网格的 pod 流量转发设置工作。
9. 在安装前启用 DevOps
在该 YAML 文件中,搜索 devops,将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置。
devops:
enabled: true # 将“false”更改为“true”。
10. 卸载方法
kubectl delete -f cluster-configuration.yaml --force
kubectl delete -f kubesphere-installer.yaml --force
#删除残余文件
vi del.sh
#!/usr/bin/env bash
function delete_sure(){
cat << eof
$(echo -e "\033[1;36mNote:\033[0m")
Delete the KubeSphere cluster, including the module kubesphere-system kubesphere-devops-system kubesphere-devops-worker kubesphere-monitoring-system kubesphere-logging-system openpitrix-system.
eof
read -p "Please reconfirm that you want to delete the KubeSphere cluster. (yes/no) " ans
while [[ "x"$ans != "xyes" && "x"$ans != "xno" ]]; do
read -p "Please reconfirm that you want to delete the KubeSphere cluster. (yes/no) " ans
done
if [[ "x"$ans == "xno" ]]; then
exit
fi
}
delete_sure
# delete ks-installer
kubectl delete deploy ks-installer -n kubesphere-system 2>/dev/null
# delete helm
for namespaces in kubesphere-system kubesphere-devops-system kubesphere-monitoring-system kubesphere-logging-system openpitrix-system kubesphere-monitoring-federated
do
helm list -n $namespaces | grep -v NAME | awk '{print $1}' | sort -u | xargs -r -L1 helm uninstall -n $namespaces 2>/dev/null
done
# delete kubefed
kubectl get cc -n kubesphere-system ks-installer -o jsonpath="{.status.multicluster}" | grep enable
if [[ $? -eq 0 ]]; then
# delete kubefed types resources
for kubefed in `kubectl api-resources --namespaced=true --api-group=types.kubefed.io -o name`
do
kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null
done
for kubefed in `kubectl api-resources --namespaced=false --api-group=types.kubefed.io -o name`
do
kubectl delete $kubefed --all 2>/dev/null
done
# delete kubefed core resouces
for kubefed in `kubectl api-resources --namespaced=true --api-group=core.kubefed.io -o name`
do
kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null
done
for kubefed in `kubectl api-resources --namespaced=false --api-group=core.kubefed.io -o name`
do
kubectl delete $kubefed --all 2>/dev/null
done
# uninstall kubefed chart
helm uninstall -n kube-federation-system kubefed 2>/dev/null
fi
helm uninstall -n kube-system snapshot-controller 2>/dev/null
# delete kubesphere deployment & statefulset
kubectl delete deployment -n kubesphere-system `kubectl get deployment -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
kubectl delete statefulset -n kubesphere-system `kubectl get statefulset -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
# delete monitor resources
kubectl delete prometheus -n kubesphere-monitoring-system k8s 2>/dev/null
kubectl delete Alertmanager -n kubesphere-monitoring-system main 2>/dev/null
kubectl delete DaemonSet -n kubesphere-monitoring-system node-exporter 2>/dev/null
kubectl delete statefulset -n kubesphere-monitoring-system `kubectl get statefulset -n kubesphere-monitoring-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null
# delete grafana
kubectl delete deployment -n kubesphere-monitoring-system grafana 2>/dev/null
kubectl --no-headers=true get pvc -n kubesphere-monitoring-system -o custom-columns=:metadata.namespace,:metadata.name | grep -E kubesphere-monitoring-system | xargs -n2 kubectl delete pvc -n 2>/dev/null
# delete pvc
pvcs="kubesphere-system|openpitrix-system|kubesphere-devops-system|kubesphere-logging-system"
kubectl --no-headers=true get pvc --all-namespaces -o custom-columns=:metadata.namespace,:metadata.name | grep -E $pvcs | xargs -n2 kubectl delete pvc -n 2>/dev/null
# delete rolebindings
delete_role_bindings() {
for rolebinding in `kubectl -n $1 get rolebindings -l iam.kubesphere.io/user-ref -o jsonpath="{.items[*].metadata.name}"`
do
kubectl -n $1 delete rolebinding $rolebinding 2>/dev/null
done
}
# delete roles
delete_roles() {
kubectl -n $1 delete role admin 2>/dev/null
kubectl -n $1 delete role operator 2>/dev/null
kubectl -n $1 delete role viewer 2>/dev/null
for role in `kubectl -n $1 get roles -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
do
kubectl -n $1 delete role $role 2>/dev/null
done
}
# remove useless labels and finalizers
for ns in `kubectl get ns -o jsonpath="{.items[*].metadata.name}"`
do
kubectl label ns $ns kubesphere.io/workspace-
kubectl label ns $ns kubesphere.io/namespace-
kubectl patch ns $ns -p '{"metadata":{"finalizers":null,"ownerReferences":null}}'
delete_role_bindings $ns
delete_roles $ns
done
# delete clusterroles
delete_cluster_roles() {
for role in `kubectl get clusterrole -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
do
kubectl delete clusterrole $role 2>/dev/null
done
for role in `kubectl get clusterroles | grep "kubesphere" | awk '{print $1}'| paste -sd " "`
do
kubectl delete clusterrole $role 2>/dev/null
done
}
delete_cluster_roles
# delete clusterrolebindings
delete_cluster_role_bindings() {
for rolebinding in `kubectl get clusterrolebindings -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"`
do
kubectl delete clusterrolebindings $rolebinding 2>/dev/null
done
for rolebinding in `kubectl get clusterrolebindings | grep "kubesphere" | awk '{print $1}'| paste -sd " "`
do
kubectl delete clusterrolebindings $rolebinding 2>/dev/null
done
}
delete_cluster_role_bindings
# delete clusters
for cluster in `kubectl get clusters -o jsonpath="{.items[*].metadata.name}"`
do
kubectl patch cluster $cluster -p '{"metadata":{"finalizers":null}}' --type=merge
done
kubectl delete clusters --all 2>/dev/null
# delete workspaces
for ws in `kubectl get workspaces -o jsonpath="{.items[*].metadata.name}"`
do
kubectl patch workspace $ws -p '{"metadata":{"finalizers":null}}' --type=merge
done
kubectl delete workspaces --all 2>/dev/null
# make DevOps CRs deletable
for devops_crd in $(kubectl get crd -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep "devops.kubesphere.io"); do
for ns in $(kubectl get ns -ojsonpath='{.items..metadata.name}'); do
for devops_res in $(kubectl get $devops_crd -n $ns -oname); do
kubectl patch $devops_res -n $ns -p '{"metadata":{"finalizers":[]}}' --type=merge
done
done
done
# delete validatingwebhookconfigurations
for webhook in ks-events-admission-validate users.iam.kubesphere.io network.kubesphere.io validating-webhook-configuration resourcesquotas.quota.kubesphere.io
do
kubectl delete validatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null
done
# delete mutatingwebhookconfigurations
for webhook in ks-events-admission-mutate logsidecar-injector-admission-mutate mutating-webhook-configuration
do
kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null
done
# delete users
for user in `kubectl get users -o jsonpath="{.items[*].metadata.name}"`
do
kubectl patch user $user -p '{"metadata":{"finalizers":null}}' --type=merge
done
kubectl delete users --all 2>/dev/null
# delete helm resources
for resource_type in `echo helmcategories helmapplications helmapplicationversions helmrepos helmreleases`; do
for resource_name in `kubectl get ${resource_type}.application.kubesphere.io -o jsonpath="{.items[*].metadata.name}"`; do
kubectl patch ${resource_type}.application.kubesphere.io ${resource_name} -p '{"metadata":{"finalizers":null}}' --type=merge
done
kubectl delete ${resource_type}.application.kubesphere.io --all 2>/dev/null
done
# delete workspacetemplates
for workspacetemplate in `kubectl get workspacetemplates.tenant.kubesphere.io -o jsonpath="{.items[*].metadata.name}"`
do
kubectl patch workspacetemplates.tenant.kubesphere.io $workspacetemplate -p '{"metadata":{"finalizers":null}}' --type=merge
done
kubectl delete workspacetemplates.tenant.kubesphere.io --all 2>/dev/null
# delete federatednamespaces in namespace kubesphere-monitoring-federated
for resource in $(kubectl get federatednamespaces.types.kubefed.io -n kubesphere-monitoring-federated -oname); do
kubectl patch "${resource}" -p '{"metadata":{"finalizers":null}}' --type=merge -n kubesphere-monitoring-federated
done
# delete crds
for crd in `kubectl get crds -o jsonpath="{.items[*].metadata.name}"`
do
if [[ $crd == *kubesphere.io ]] || [[ $crd == *kubefed.io ]] ; then kubectl delete crd $crd 2>/dev/null; fi
done
# delete relevance ns
for ns in kube-federation-system kubesphere-alerting-system kubesphere-controls-system kubesphere-devops-system kubesphere-devops-worker kubesphere-logging-system kubesphere-monitoring-system kubesphere-monitoring-federated openpitrix-system kubesphere-system
do
kubectl delete ns $ns 2>/dev/null
done
#执行删除
sh del.sh
十二、 GitLab 安装
1. 单独准备服务器,采用 Docker 安装
docker search gitlab
docker pull gitlab/gitlab-ce
2. 准备 docker-compose.yml 文件
mkdir -p /data/git
vim /data/git/docker-compose.yml
version: '3.1'
services:
gitlab:
image: 'gitlab/gitlab-ce:latest'
container_name: gitlab
restart: always
environment:
GITLAB_OMNIBUS_CONFIG: |
external_url 'http://10.1.100.225:8929'#自己安装git的服务器IP
gitlab_rails['gitlab_shell_ssh_port'] = 2224
ports:
- '8929:8929'
- '2224:2224'
volumes:
- './config:/etc/gitlab'
- './logs:/var/log/gitlab'
- './data:/var/opt/gitlab'
3. 启动容器(需要稍等很久……)
cd /data/git
docker-compose up -d
4. 访问 GitLab 首页
http://10.1.100.225:8929
5. 查看 root 用户初始密码
docker exec -it gitlab cat /etc/gitlab/initial_root_password
6. 第一次登录网页,需要修改密码 Password
** 右上角 >>**Administrator>Preferences>Password
十三、DevOps 初始化环境
1.linux 系统 安装 Jenkins、jdk 、maven
1. 下载地址
JDK 包下载地址
https://www.oracle.com/java/technologies/downloads/
MAven 下载地址
https://maven.apache.org/download.cgi
2. 安装 jdk maven
tar -zxvf jdk-8*.tar.gz -C /usr/local/
tar -zxvf apache-maven-*.tar.gz -C /usr/local/
cd /usr/local
mv apache-maven*/ maven
mv jdk1.8*/ jdk
2.1 编辑 maven 配置
vim /usr/local/maven/conf/settings.xml
<!--#maven配置阿里云仓库,在160行插入-->
<mirror>
<id>nexus-aliyun</id>
<mirrorOf>central</mirrorOf>
<name>Nexus aliyun</name>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
</mirror>
<!--#maven配置jdk,在252行插入-->
<profile>
<id>jdk1.8</id>
<activation>
<activeByDefault>true</activeByDefault>
<jdk>1.8</jdk>
</activation>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion>
</properties>
</profile>
<!--#maven配置jdk,在257行插入-->
<activeProfiles>
<activeProfile>jdk1.8</activeProfile>
</activeProfiles>
3. 安装 jenkins
3.1 下载
docker pull jenkins/jenkins:2.319.1-lts
3.2 创建 yaml
mkdir -p /data/jenkins/
cd /data/jenkins/
vim /data/jenkins/docker-compose.yml
version: "3.1"
services:
jenkins:
image: jenkins/jenkins
container_name: jenkins
ports:
- 8080:8080
- 50000:50000
volumes:
- ./data/:/var/jenkins_home/
- /var/run/docker.sock:/var/run/docker.sock
- /usr/bin/docker:/usr/bin/docker
- /etc/docker/daemon.json:/etc/docker/daemon.json
3.3 启动 jenkins
#修改Jenkins用户权限
cd /var/run
chown root:root docker.sock
#其他用户有读和写权限
chmod o+rw docker.sock
cd /data/jenkins/
docker-compose up -d
#授权
chmod 777 /data/jenkins/data/
cat /data/jenkins/data/hudson.model.UpdateCenter.xml
#重新启动Jenkins容器后,由于Jenkins需要下载大量内容,但是由于默认下载地址下载速度较慢,
#需要重新设置下载地址为国内镜像站# 清华大学的插件源也可以
# 修改数据卷中的hudson.model.UpdateCenter.xml文件
# 将下载地址替换为http://mirror.esuni.jp/jenkins/updates/update-center.json
# 清华大学的插件源也可以
#https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
#重启
docker-compose restart
#查看日志
docker logs -f jenkins
3.4 访问页面,安装插件
http://10.1.100.225:8080
1. 输入密码 2. 选择插件来安装 3. 点击安装
4.jenkins 插件安装
中文界面>系统管理>插件管理>可选插件>搜索插件
英文界面> Manage Jenkins–Manage Plugins-Available>搜索插件
Locale
Localization
Git Parameter
Publish Over SSH
5. 配置 jenkins
mv /usr/local/maven/ /data/jenkins/data/
mv /usr/local/jdk/ /data/jenkins/data/
5.1 加载本地 jdk
Dashboard > 系统管理 > 全局工具配置 > Add JDK > 去掉对钩 (√)自动安装
NAME
jdk8
JAVA_HOME
/var/jenkins_home/jdk/
5.1 加载本地 maven
Dashboard > 系统管理 > 全局工具配置 > Add Maven > 去掉对钩 (√)自动安装
NAME
maven
JAVA_HOME
/var/jenkins_home/maven/
Save Apply
保存 应用
运行 mvn 测试
mvn help:system
3.jenkins 拉取测试
系统管理 > 系统配置 > Publish over SSH>SSH Servers>Add
#自定义项目名称
name
test
#主机 IP
Hostname
10.1.100.25
#主机用户名
Username
root
#拉取项目路径
Remote Directory
/data/work/mytest
点击高级
√ Use password authentication, or use a different key
#输入服务器密码
Passphrase / Password
xxxx
#点击 测试
Test Configuration
Save Apply
保存 应用
4.Jenkins 服务器设置免密登入 k8s-mast 服务器
#Jenkins 服务器 - 进入 jenkins 容器
docker exec -it jenkins bash
#进入 jenkins 容器 - 生成免密登录公私钥,根据提示按回车
ssh-keygen -t rsa
#进入 jenkins 容器 - 查看 jenkins 秘钥
cat /var/jenkins_home/.ssh/id_rsa.pub
#k8s-mast 服务器中 authorized_keys 加入 Jenkins 服务器秘钥
echo “xxxxxx” >> /root/.ssh/authorized_keys
十四、开发环境部署 IDEA
工具下载:
链接:https://pan.baidu.com/s/1Jkyh_kgrT2o388Xiujbdeg?pwd=b7rx
提取码:b7rx
1. windows 配置 maven 和 jdk
https://blog.csdn.net/weixin_46565024/article/details/122758111
2. IDEA 简单得项目创建
File>New>Project
Spring Initializr>Next
Type(选择Maven)>Java Version (选择8) > Next
Web> 勾选√Spring Web> Next>Finish
(责任编辑:IT)
一、基础配置 1. 修改主机名 1.hostnamectl set-hostname k8s-master01 2.hostnamectl set-hostname k8s-master02 3.hostnamectl set-hostname k8s-master03 4.hostnamectl set-hostname k8s-node01 5.hostnamectl set-hostname k8s-node02 2. 添加 主机名与 IP 地址解析 cat > /etc/hosts <<EOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.1.62 lookup apiserver.cluster.local 192.168.1.60 k8s-master01 192.168.1.61 k8s-master02 192.168.1.62 k8s-master03 192.168.1.63 k8s-node01 192.168.1.64 k8s-node02 EOF 3. 升级服务器内核,时间同步,关闭防火墙,重启服务器 #添加访问互联路由 cat > /etc/resolv.conf <<EOF nameserver 114.114.114.114 nameserver 8.8.8.8 EOF cat /etc/resolv.conf # ssh连接Linux比较慢 #sed -i "s|#UseDNS yes|UseDNS no|" /etc/ssh/sshd_config #sed -i "s|GSSAPIAuthentication yes|GSSAPIAuthentication no|" /etc/ssh/sshd_config #设置为阿里云yum源 rm -rf /etc/yum.repos.d/bak && mkdir -p /etc/yum.repos.d/bak && mv /etc/yum.repos.d/* /etc/yum.repos.d/bak curl -o /etc/yum.repos.d/CentOS-7.repo http://mirrors.aliyun.com/repo/Centos-7.repo yum clean all && yum makecache cd /etc/yum.repos.d #CentOS7使用/etc/rc.d/rc.local设置开机自动启动 chmod +x /etc/rc.d/rc.local #安装依赖包 yum -y install vim net-tools lrzsz unzip gcc telnet wget sshpass ntpdate ntp curl yum -y install conntrack ipvsadm ipset jq iptables sysstat libseccomp git #时间同步 echo '*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com >/dev/null 2>&1'>/var/spool/cron/root && crontab -l #设置防火墙为 Iptables 并设置空规则 systemctl stop firewalld && systemctl disable firewalld yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save #关闭 SELINUX swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config #调整内核参数,对于 K8S cat > /etc/sysctl.d/kubernetes.conf <<EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 #net.ipv4.tcp_tw_recycle=0 vm.swappiness=0 # 禁止使用 swap 空间,只有当系统 OOM 时才允许使用它 vm.overcommit_memory=1 # 不检查物理内存是否够用 vm.panic_on_oom=0 # 开启 OOM fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 fs.file-max=52706963 fs.nr_open=52706963 net.ipv6.conf.all.disable_ipv6=1 net.netfilter.nf_conntrack_max=2310720 EOF modprobe ip_vs_rr && modprobe br_netfilter && sysctl -p /etc/sysctl.d/kubernetes.conf #关闭系统不需要服务 systemctl stop postfix && systemctl disable postfix 4. 升级内核,重启服务器 rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org yum -y install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm yum --enablerepo="elrepo-kernel" -y install kernel-lt.x86_64 awk -F \' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg grub2-set-default "CentOS Linux (5.4.225-1.el7.elrepo.x86_64) 7 (Core)" #grub2-set-default 'CentOS Linux (4.4.222-1.el7.elrepo.x86_64) 7 (Core)' #重启服务器 reboot ################################ 二、sealos 部署 k8s-v1.19 1. 安装 sealos3.3 #添加访问互联路由 cat > /etc/resolv.conf <<EOF nameserver 8.8.8.8 nameserver 114.114.114.114 nameserver 223.5.5.5 EOF cat /etc/resolv.conf #时间同步 ntpdate ntp1.aliyun.com wget -c https://github.com/fanux/sealos/releases/download/v3.3.8/sealos tar zxvf sealos*.tar.gz sealos && chmod +x sealos && mv sealos /usr/bin sealos version #时间同步 ntpdate ntp1.aliyun.com 2. 离线安装 k8s 1.19 链接:https://pan.baidu.com/s/1F9sZoHBX1K1ihBP9rZSHBQ?pwd=jood 提取码:jood #安装 sealos init --passwd 1qaz@WSX \ --master 192.168.1.60 \ --master 192.168.1.61 \ --master 192.168.1.62 \ --node 192.168.1.63 \ --node 192.168.1.64 \ --pkg-url /root/kube1.19.16.tar.gz \ --version v1.19.16 3. 验证集群 kubectl get nodes kubectl get pod -A #配置kubectl自动补全 yum install -y bash-completion source /usr/share/bash-completion/bash_completion source <(kubectl completion bash) echo "source <(kubectl completion bash)" >> /etc/profile #查看污点 kubectl describe node |grep -i taints #去除污点 #kubectl taint node k8s-master02 node-role.kubernetes.io/master:NoSchedule- #kubectl taint node k8s-master03 node-role.kubernetes.io/master:NoSchedule- 4.sealos3.3 常用命令 #添加 node 节点: sealos join --node 192.168.1.63,192.168.1.64 #添加master sealos join -master 192.168.1.61,192.168.1.62 #删除 node 节点: sealos clean --node 192.168.1.63,192.168.1.64 #删除 master 节点: sealos clean --master 192.168.1.61,192.168.1.62 #重置集群 sealos clean --all -f 5. 安装 top 命令 cat > /root/top.yaml <<EOF apiVersion: v1 kind: ServiceAccount metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: k8s-app: metrics-server rbac.authorization.k8s.io/aggregate-to-admin: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-view: "true" name: system:aggregated-metrics-reader rules: - apiGroups: - metrics.k8s.io resources: - pods - nodes verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: k8s-app: metrics-server name: system:metrics-server rules: - apiGroups: - "" resources: - pods - nodes - nodes/stats - namespaces - configmaps verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: k8s-app: metrics-server name: metrics-server-auth-reader namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: k8s-app: metrics-server name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: k8s-app: metrics-server name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: v1 kind: Service metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system spec: ports: - name: https port: 443 protocol: TCP targetPort: https selector: k8s-app: metrics-server --- apiVersion: apps/v1 kind: Deployment metadata: labels: k8s-app: metrics-server name: metrics-server namespace: kube-system spec: selector: matchLabels: k8s-app: metrics-server strategy: rollingUpdate: maxUnavailable: 0 template: metadata: labels: k8s-app: metrics-server spec: containers: - args: - --cert-dir=/tmp - --kubelet-insecure-tls - --secure-port=4443 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-use-node-status-port #这里可以自己把metrics-server做到自己的阿里云镜像里面,并把下面替换成自己的镜像地址 image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/metrics-server:v0.4.3 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: /livez port: https scheme: HTTPS periodSeconds: 10 name: metrics-server ports: - containerPort: 4443 name: https protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /readyz port: https scheme: HTTPS periodSeconds: 10 securityContext: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 volumeMounts: - mountPath: /tmp name: tmp-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical serviceAccountName: metrics-server volumes: - emptyDir: {} name: tmp-dir --- apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: labels: k8s-app: metrics-server name: v1beta1.metrics.k8s.io spec: group: metrics.k8s.io groupPriorityMinimum: 100 insecureSkipTLSVerify: true service: name: metrics-server namespace: kube-system version: v1beta1 versionPriority: 100 EOF kubectl apply -f /root/top.yaml 三、部署 nfs 1. 服务端 #添加访问互联路由 cat > /etc/resolv.conf <<EOF nameserver 114.114.114.114 nameserver 8.8.8.8 EOF # 我们这里在192.168.1.60上安装(在生产中,大家要提供作好NFS-SERVER环境的规划) yum -y install nfs-utils # 创建NFS挂载目录 mkdir /nfs_dir chown nobody.nobody /nfs_dir # 修改NFS-SERVER配置 echo '/nfs_dir *(rw,sync,no_root_squash)' > /etc/exports # 重启服务 systemctl restart rpcbind.service systemctl restart nfs-utils.service systemctl restart nfs-server.service # 增加NFS-SERVER开机自启动 systemctl enable rpcbind.service systemctl enable nfs-utils.service systemctl enable nfs-server.service # 验证NFS-SERVER是否能正常访问 #showmount -e 192.168.1.60 2. 客户端 #需要挂载的服务器执行 mkdir /nfs_dir yum install nfs-utils -y #挂载 mount 192.168.1.60:/nfs_dir /nfs_dir #添加开机挂载 echo "mount 192.168.1.60:/nfs_dir /nfs_dir" >> /etc/rc.local cat /etc/rc.local 四、部署 StorageClass 1. 创建 nfs-sc.yaml cat > /root/nfs-sc.yaml <<EOF apiVersion: v1 kind: ServiceAccount metadata: name: nfs-client-provisioner namespace: kube-system --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nfs-client-provisioner-runner rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: run-nfs-client-provisioner subjects: - kind: ServiceAccount name: nfs-client-provisioner namespace: kube-system roleRef: kind: ClusterRole name: nfs-client-provisioner-runner apiGroup: rbac.authorization.k8s.io --- kind: Deployment apiVersion: apps/v1 metadata: name: nfs-provisioner-01 namespace: kube-system spec: replicas: 1 strategy: type: Recreate selector: matchLabels: app: nfs-provisioner-01 template: metadata: labels: app: nfs-provisioner-01 spec: serviceAccountName: nfs-client-provisioner containers: - name: nfs-client-provisioner #老版本插件使用jmgao1983/nfs-client-provisioner:latest # image: jmgao1983/nfs-client-provisioner:latest image: vbouchaud/nfs-client-provisioner:latest imagePullPolicy: IfNotPresent volumeMounts: - name: nfs-client-root mountPath: /persistentvolumes env: - name: PROVISIONER_NAME value: nfs-provisioner-01 # 此处供应者名字供storageclass调用 - name: NFS_SERVER value: 192.168.1.60 # 填入NFS的地址 - name: NFS_PATH value: /nfs_dir # 填入NFS挂载的目录 volumes: - name: nfs-client-root nfs: server: 192.168.1.60 # 填入NFS的地址 path: /nfs_dir # 填入NFS挂载的目录 --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: nfs-boge provisioner: nfs-provisioner-01 # Supported policies: Delete、 Retain , default is Delete reclaimPolicy: Retain EOF #创建 kubectl apply -f /root/nfs-sc.yaml #查看 kubectl -n kube-system get pod kubectl get sc 五、harbor 仓库搭建 1. 安装 #目录/root上传文件docker-compose和harbor-offline-installer-v1.2.0.tgz mv /root/docker-compose /usr/local/bin/ chmod a+x /usr/local/bin/docker-compose ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose tar -zxvf harbor-offline-installer-v2.4.1.tgz mv harbor /usr/local/ cd /usr/local/harbor/ cp harbor.yml.tmpl harbor.yml sed -i 's/hostname: reg.mydomain.com/hostname: 192.168.1.77/g' harbor.yml sed -i 's/https/#https/g' harbor.yml sed -i 's/certificate/#certificate/g' harbor.yml sed -i 's/private_key/#private_key/g' harbor.yml #数据库目录 mkdir /data cat /etc/docker/daemon.json { "registry-mirrors": ["https://nr240upq.mirror.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn", "http://hub-mirror.c.163.com"], "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", "log-opts": { "max-size": "100m" }, "insecure-registries": ["192.168.1.77:80"] } systemctl daemon-reload && systemctl restart docker #安装 ./install.sh ## 重启harbor cd /usr/local/harbor/ docker-compose down -v docker-compose up -d docker ps|grep harbor netstat -ntlp 2. 需要访问仓库的其他节点的 daemon.json 添加如下内容 ##------------------- vim /etc/docker/daemon.json "registry-mirrors": ["https://nr240upq.mirror.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn"], "insecure-registries": ["192.168.1.77:80"], ##------------------- #重启 systemctl daemon-reload && systemctl restart docker 3. 节点使用仓库 #登入仓库网站 docker login -u admin -p Harbor12345 192.168.1.77:80 #下载镜像 docker pull daocloud.io/library/nginx:1.9.1 #给镜像打上标签 docker tag daocloud.io/library/nginx:1.9.1 192.168.1.77:80/library/nginx:1.9.1 #镜像上传 docker push 192.168.1.77:80/library/nginx:1.9.1 #删除镜像 docker rmi 192.168.1.77:80/library/nginx:1.9.1 #将镜像保存为本地tar文件, docker save k8s.gcr.io/coredns:1.7.0 > /root/coredns-v1.7.0.tar #使用load加载tar文件 docker load -i /root/coredns-v1.7.0.tar 4. 批量打包上传 harbor 镜像 cd /root #查看服务器镜像名称 docker images | awk 'NR!=1{print $1":"$2}' > 01-image-old.txt && cat 01-image-old.txt # /换成- rm -rf 02-image-sed.txt && cp 01-image-old.txt 02-image-sed.txt && sed -i "s|/|-|g" 02-image-sed.txt && cat /root/02-image-sed.txt #打标签harbor仓库 vim /root/03-tar-image.sh ##################################################### #!/bin/sh old=/root/01-image-old.txt new=/root/02-image-sed.txt l=$(cat /root/01-image-old.txt| wc -l) for ((i=1 ; i<=$l ; i++)) do a=$(sed -n "$i"p $old) b=$(sed -n "$i"p $new) #echo "update xxxx set uid='$a' where uid='$b';" docker tag $a 192.168.1.77:80/library/$b done ##################################################### #运行打仓库标签 bash /root/03-tar-image.sh docker images |grep library #查看打标harbor仓库images名称 docker images |grep 192.168.1.77 | awk '{print $1":"$2}' > 04-tar-image.txt && cat 04-tar-image.txt #上传到harbor仓库 for h in `cat 04-tar-image.txt`; do docker push $h; done #删除打标镜像 for d in `cat 04-tar-image.txt`; do docker rmi $d; done docker images |grep library #删除创建的文件 rm -rf /root/0*txt 03-tar-image.sh 六、kuboard 界面管理 1. 下载地址 curl -o kuboard-v3.yaml https://addons.kuboard.cn/kuboard/kuboard-v3-storage-class.yaml 2. 编辑 yaml #编辑 kuboard-v3.yaml 文件中的配置,该部署文件中,有1处配置必须修改:storageClassName volumeClaimTemplates: - metadata: name: data spec: # 请填写一个有效的 StorageClass name storageClassName: nfs-boge accessModes: [ "ReadWriteMany" ] resources: requests: storage: 5Gi 3. 执行 kubectl create -f kuboard-v3.yaml kubectl get pod -n kuboard ############################################ #访问 http://192.168.1.60:30080/ 输入初始用户名和密码,并登录 用户名: admin 密码: Kuboard123 ############################################# #查看错误 journalctl -f -u kubelet.service 七、helm3 安装 1.helm 包下载地址 wget https://get.helm.sh/helm-v3.6.1-linux-amd64.tar.gz 2. 安装 helm #解压 && 移动到 /usr/bin 目录下: tar -xvf helm-v3.6.1-linux-amd64.tar.gz && cd linux-amd64/ && mv helm /usr/bin #查看版本 helm version 3. 配置仓库 #添加公用的仓库 helm repo add incubator https://charts.helm.sh/incubator helm repo add bitnami https://charts.bitnami.com/bitnami # 配置helm微软源地址 helm repo add stable http://mirror.azure.cn/kubernetes/charts # 配置helm阿里源地址 helm repo add aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts helm repo add google https://kubernetes-charts.storage.googleapis.com helm repo add jetstack https://charts.jetstack.io # 查看仓库 helm repo list # 更新仓库 helm repo update # 删除仓库 #helm repo remove aliyun # helm list 八、haproxy+keepalived+ingress 1. 部署阿里云 ingress mkdir -p /data/k8s/ cd /data/k8s/ cat > /data/k8s/aliyun-ingress-nginx.yaml <<EOF apiVersion: v1 kind: Namespace metadata: name: ingress-nginx labels: app: ingress-nginx --- apiVersion: v1 kind: ServiceAccount metadata: name: nginx-ingress-controller namespace: ingress-nginx labels: app: ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-controller labels: app: ingress-nginx rules: - apiGroups: - "" resources: - configmaps - endpoints - nodes - pods - secrets - namespaces - services verbs: - get - list - watch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses/status verbs: - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "" resources: - configmaps resourceNames: - "ingress-controller-leader-nginx" verbs: - get - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: nginx-ingress-controller labels: app: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nginx-ingress-controller subjects: - kind: ServiceAccount name: nginx-ingress-controller namespace: ingress-nginx --- apiVersion: v1 kind: Service metadata: labels: app: ingress-nginx name: nginx-ingress-lb namespace: ingress-nginx spec: # DaemonSet need: # ---------------- type: ClusterIP # ---------------- # Deployment need: # ---------------- # type: NodePort # ---------------- ports: - name: http port: 80 targetPort: 80 protocol: TCP - name: https port: 443 targetPort: 443 protocol: TCP - name: metrics port: 10254 protocol: TCP targetPort: 10254 selector: app: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: nginx-configuration namespace: ingress-nginx labels: app: ingress-nginx data: keep-alive: "75" keep-alive-requests: "100" upstream-keepalive-connections: "10000" upstream-keepalive-requests: "100" upstream-keepalive-timeout: "60" allow-backend-server-header: "true" enable-underscores-in-headers: "true" generate-request-id: "true" http-redirect-code: "301" ignore-invalid-headers: "true" log-format-upstream: '{"@timestamp": "$time_iso8601","remote_addr": "$remote_addr","x-forward-for": "$proxy_add_x_forwarded_for","request_id": "$req_id","remote_user": "$remote_user","bytes_sent": $bytes_sent,"request_time": $request_time,"status": $status,"vhost": "$host","request_proto": "$server_protocol","path": "$uri","request_query": "$args","request_length": $request_length,"duration": $request_time,"method": "$request_method","http_referrer": "$http_referer","http_user_agent": "$http_user_agent","upstream-sever":"$proxy_upstream_name","proxy_alternative_upstream_name":"$proxy_alternative_upstream_name","upstream_addr":"$upstream_addr","upstream_response_length":$upstream_response_length,"upstream_response_time":$upstream_response_time,"upstream_status":$upstream_status}' max-worker-connections: "65536" worker-processes: "2" proxy-body-size: 20m proxy-connect-timeout: "10" proxy_next_upstream: error timeout http_502 reuse-port: "true" server-tokens: "false" ssl-ciphers: ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA ssl-protocols: TLSv1 TLSv1.1 TLSv1.2 ssl-redirect: "false" worker-cpu-affinity: auto --- kind: ConfigMap apiVersion: v1 metadata: name: tcp-services namespace: ingress-nginx labels: app: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: udp-services namespace: ingress-nginx labels: app: ingress-nginx --- apiVersion: apps/v1 kind: DaemonSet metadata: name: nginx-ingress-controller namespace: ingress-nginx labels: app: ingress-nginx annotations: component.version: "v0.30.0" component.revision: "v1" spec: # Deployment need: # ---------------- # replicas: 1 # ---------------- selector: matchLabels: app: ingress-nginx template: metadata: labels: app: ingress-nginx annotations: prometheus.io/port: "10254" prometheus.io/scrape: "true" scheduler.alpha.kubernetes.io/critical-pod: "" spec: # DaemonSet need: # ---------------- hostNetwork: true # ---------------- serviceAccountName: nginx-ingress-controller priorityClassName: system-node-critical affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - ingress-nginx topologyKey: kubernetes.io/hostname weight: 100 nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: NotIn values: - virtual-kubelet containers: - name: nginx-ingress-controller image: registry.cn-beijing.aliyuncs.com/acs/aliyun-ingress-controller:v0.30.0.2-9597b3685-aliyun args: - /nginx-ingress-controller - --configmap=$(POD_NAMESPACE)/nginx-configuration - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb - --annotations-prefix=nginx.ingress.kubernetes.io - --enable-dynamic-certificates=true - --v=2 securityContext: allowPrivilegeEscalation: true capabilities: drop: - ALL add: - NET_BIND_SERVICE runAsUser: 101 env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - name: http containerPort: 80 - name: https containerPort: 443 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 readinessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 # resources: # limits: # cpu: "1" # memory: 2Gi # requests: # cpu: "1" # memory: 2Gi volumeMounts: - mountPath: /etc/localtime name: localtime readOnly: true volumes: - name: localtime hostPath: path: /etc/localtime type: File nodeSelector: boge/ingress-controller-ready: "true" tolerations: - operator: Exists initContainers: - command: - /bin/sh - -c - | mount -o remount rw /proc/sys sysctl -w net.core.somaxconn=65535 sysctl -w net.ipv4.ip_local_port_range="1024 65535" sysctl -w fs.file-max=1048576 sysctl -w fs.inotify.max_user_instances=16384 sysctl -w fs.inotify.max_user_watches=524288 sysctl -w fs.inotify.max_queued_events=16384 image: registry.cn-beijing.aliyuncs.com/acs/busybox:v1.29.2 imagePullPolicy: Always name: init-sysctl securityContext: privileged: true procMount: Default --- ## Deployment need for aliyun'k8s: #apiVersion: v1 #kind: Service #metadata: # annotations: # service.beta.kubernetes.io/alibaba-cloud-loadbalancer-id: "lb-xxxxxxxxxxxxxxxxxxx" # service.beta.kubernetes.io/alibaba-cloud-loadbalancer-force-override-listeners: "true" # labels: # app: nginx-ingress-lb # name: nginx-ingress-lb-local # namespace: ingress-nginx #spec: # externalTrafficPolicy: Local # ports: # - name: http # port: 80 # protocol: TCP # targetPort: 80 # - name: https # port: 443 # protocol: TCP # targetPort: 443 # selector: # app: ingress-nginx # type: LoadBalancer EOF kubectl apply -f /data/k8s/aliyun-ingress-nginx.yaml 2. 节点打标签 #允许节点打标签 kubectl label node k8s-master01 boge/ingress-controller-ready=true kubectl label node k8s-master02 boge/ingress-controller-ready=true kubectl label node k8s-master03 boge/ingress-controller-ready=true #删除标签 #kubectl label node k8s-master01 boge/ingress-controller-ready=true --overwrite #kubectl label node k8s-master02 boge/ingress-controller-ready=true --overwrite #kubectl label node k8s-master03 boge/ingress-controller-ready=true --overwrite 3.haproxy+keepalived 部署 3.0 部署 yum install haproxy keepalived -y #重启程序 systemctl restart haproxy.service systemctl restart keepalived.service # 查看运行状态 systemctl status haproxy.service systemctl status keepalived.service #开机自启动 systemctl enable keepalived.service systemctl enable haproxy.service 3.1 修改配置 haproxy vim /etc/haproxy/haproxy.cfg ################################################### listen ingress-http bind 0.0.0.0:80 mode tcp option tcplog option dontlognull option dontlog-normal balance roundrobin server 192.168.1.60 192.168.1.60:80 check inter 2000 fall 2 rise 2 weight 1 server 192.168.1.61 192.168.1.61:80 check inter 2000 fall 2 rise 2 weight 1 server 192.168.1.62 192.168.1.62:80 check inter 2000 fall 2 rise 2 weight 1 listen ingress-https bind 0.0.0.0:443 mode tcp option tcplog option dontlognull option dontlog-normal balance roundrobin server 192.168.1.60 192.168.1.60:443 check inter 2000 fall 2 rise 2 weight 1 server 192.168.1.61 192.168.1.61:443 check inter 2000 fall 2 rise 2 weight 1 server 192.168.1.62 192.168.1.62:443 check inter 2000 fall 2 rise 2 weight 1 3.2 A 机器修改 keepalived 配置 cat > /etc/keepalived/keepalived.conf <<EOF global_defs { router_id lb-master } vrrp_script check-haproxy { script "killall -0 haproxy" interval 5 weight -60 } vrrp_instance VI-kube-master { state MASTER priority 120 unicast_src_ip 192.168.1.63 #本机ip unicast_peer { 192.168.1.64 #另一台机器ip } dont_track_primary interface ens33 # 注意这里的网卡名称修改成你机器真实的内网网卡名称,可用命令ip addr查看 virtual_router_id 111 advert_int 3 track_script { check-haproxy } virtual_ipaddress { 192.168.1.100 #vip 地址 } } EOF 3.3 B 机器修改 keepalived 配置 cat > /etc/keepalived/keepalived.conf <<EOF global_defs { router_id lb-master } vrrp_script check-haproxy { script "killall -0 haproxy" interval 5 weight -60 } vrrp_instance VI-kube-master { state MASTER priority 120 unicast_src_ip 192.168.1.64 #本机ip unicast_peer { 192.168.1.63 #另一台机器ip } dont_track_primary interface ens33 # 注意这里的网卡名称修改成你机器真实的内网网卡名称,可用命令ip addr查看 virtual_router_id 111 advert_int 3 track_script { check-haproxy } virtual_ipaddress { 192.168.1.100 #vip 地址 } } EOF 3.4 重启 #重启程序 systemctl restart haproxy.service systemctl restart keepalived.service # 查看运行状态 systemctl status haproxy.service systemctl status keepalived.service 4. 部署 nginx-ingress cat > /root/nginx-ingress.yaml <<EOF apiVersion: v1 kind: Service metadata: namespace: test name: nginx labels: app: nginx spec: ports: - port: 80 protocol: TCP targetPort: 80 selector: app: nginx --- apiVersion: apps/v1 kind: Deployment metadata: namespace: test name: nginx labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx ports: - containerPort: 80 --- apiVersion: extensions/v1beta1 kind: Ingress metadata: namespace: test name: nginx-ingress spec: rules: - host: nginx.boge.com http: paths: - backend: serviceName: nginx servicePort: 80 path: / EOF 5. 测试 nginx-ingress kubectl apply -f /root/nginx-ingress.yaml #查看创建的ingress资源 kubectl get ingress -A #服务器新增域名解析 echo "192.168.1.100 nginx.boge.com" >> /etc/hosts # 我们在其它节点上,加下本地hosts,来测试下效果 20.6.1.226 nginx.boge.com #测试 curl nginx.boge.com 九、elk 日志监控 1. 创建测试 tomcat cat > 01-tomcat-test.yaml <<EOF apiVersion: apps/v1 kind: Deployment metadata: labels: app: tomcat name: tomcat spec: replicas: 1 selector: matchLabels: app: tomcat template: metadata: labels: app: tomcat spec: tolerations: - key: "node-role.kubernetes.io/master" effect: "NoSchedule" containers: - name: tomcat image: "tomcat:7.0" env: # 注意点一,添加相应的环境变量(下面收集了两块日志1、stdout 2、/usr/local/tomcat/logs/catalina.*.log) - name: aliyun_logs_tomcat-syslog # 如日志发送到es,那index名称为 tomcat-syslog value: "stdout" - name: aliyun_logs_tomcat-access # 如日志发送到es,那index名称为 tomcat-access value: "/usr/local/tomcat/logs/catalina.*.log" volumeMounts: # 注意点二,对pod内要收集的业务日志目录需要进行共享,可以收集多个目录下的日志文件 - name: tomcat-log mountPath: /usr/local/tomcat/logs volumes: - name: tomcat-log emptyDir: {} EOF kubectl apply -f 01-tomcat-test.yaml 2. 部署 elasticsearch cat > 02-elasticsearch.6.8.13-statefulset.yaml <<EOF apiVersion: apps/v1 kind: StatefulSet metadata: labels: addonmanager.kubernetes.io/mode: Reconcile k8s-app: elasticsearch-logging version: v6.8.13 name: elasticsearch-logging namespace: logging spec: replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: k8s-app: elasticsearch-logging version: v6.8.13 serviceName: elasticsearch-logging template: metadata: labels: k8s-app: elasticsearch-logging version: v6.8.13 spec: # nodeSelector: # esnode: "true" ## 注意给想要运行到的node打上相应labels containers: - env: - name: NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: cluster.name value: elasticsearch-logging-0 - name: ES_JAVA_OPTS value: "-Xms512m -Xmx512m" image: elastic/elasticsearch:6.8.13 name: elasticsearch-logging ports: - containerPort: 9200 name: db protocol: TCP - containerPort: 9300 name: transport protocol: TCP volumeMounts: - mountPath: /usr/share/elasticsearch/data name: elasticsearch-logging dnsConfig: options: - name: single-request-reopen initContainers: - command: - /bin/sysctl - -w - vm.max_map_count=262144 image: busybox imagePullPolicy: IfNotPresent name: elasticsearch-logging-init resources: {} securityContext: privileged: true - name: fix-permissions image: busybox command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] securityContext: privileged: true volumeMounts: - name: elasticsearch-logging mountPath: /usr/share/elasticsearch/data volumes: - name: elasticsearch-logging hostPath: path: /esdata --- apiVersion: v1 kind: Service metadata: labels: k8s-app: elasticsearch-logging name: elasticsearch namespace: logging spec: ports: - port: 9200 protocol: TCP targetPort: db selector: k8s-app: elasticsearch-logging type: ClusterIP kubectl apply -f 02-elasticsearch.6.8.13-statefulset.yaml 3. 部署 kibana cat > 03-kibana.6.8.13.yaml <<EOF apiVersion: apps/v1 kind: Deployment metadata: name: kibana namespace: logging labels: app: kibana spec: selector: matchLabels: app: kibana template: metadata: labels: app: kibana spec: containers: - name: kibana image: elastic/kibana:6.8.13 resources: limits: cpu: 1000m requests: cpu: 100m env: - name: ELASTICSEARCH_URL value: http://elasticsearch:9200 ports: - containerPort: 5601 --- apiVersion: v1 kind: Service metadata: name: kibana namespace: logging labels: app: kibana spec: ports: - port: 5601 protocol: TCP targetPort: 5601 type: ClusterIP selector: app: kibana --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: kibana namespace: logging spec: rules: - host: kibana.boge.com http: paths: - path: / backend: serviceName: kibana servicePort: 5601 kubectl apply -f 03-kibana.6.8.13.yaml 4. 部署 log-pilot cat > 04-log-pilot.yml <<EOF apiVersion: apps/v1 kind: DaemonSet metadata: name: log-pilot namespace: logging labels: app: log-pilot # 设置期望部署的namespace spec: selector: matchLabels: app: log-pilot updateStrategy: type: RollingUpdate template: metadata: labels: app: log-pilot annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: # 是否允许部署到Master节点上 #tolerations: #- key: node-role.kubernetes.io/master # effect: NoSchedule containers: - name: log-pilot # 版本请参考https://github.com/AliyunContainerService/log-pilot/releases image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat resources: limits: memory: 500Mi requests: cpu: 200m memory: 200Mi env: - name: "NODE_NAME" valueFrom: fieldRef: fieldPath: spec.nodeName ##-------------------------------- # - name: "LOGGING_OUTPUT" # value: "logstash" # - name: "LOGSTASH_HOST" # value: "logstash-g1" # - name: "LOGSTASH_PORT" # value: "5044" ##-------------------------------- - name: "LOGGING_OUTPUT" value: "elasticsearch" ## 请确保集群到ES网络可达 - name: "ELASTICSEARCH_HOSTS" value: "elasticsearch:9200" ## 配置ES访问权限 #- name: "ELASTICSEARCH_USER" # value: "{es_username}" #- name: "ELASTICSEARCH_PASSWORD" # value: "{es_password}" ##-------------------------------- ## https://github.com/AliyunContainerService/log-pilot/blob/master/docs/filebeat/docs.md ## to file need configure 1 # - name: LOGGING_OUTPUT # value: file # - name: FILE_PATH # value: /tmp # - name: FILE_NAME # value: filebeat.log volumeMounts: - name: sock mountPath: /var/run/docker.sock - name: root mountPath: /host readOnly: true - name: varlib mountPath: /var/lib/filebeat - name: varlog mountPath: /var/log/filebeat - name: localtime mountPath: /etc/localtime readOnly: true ## to file need configure 2 # - mountPath: /tmp # name: mylog livenessProbe: failureThreshold: 3 exec: command: - /pilot/healthz initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 2 securityContext: capabilities: add: - SYS_ADMIN terminationGracePeriodSeconds: 30 volumes: - name: sock hostPath: path: /var/run/docker.sock - name: root hostPath: path: / - name: varlib hostPath: path: /var/lib/filebeat type: DirectoryOrCreate - name: varlog hostPath: path: /var/log/filebeat type: DirectoryOrCreate - name: localtime hostPath: path: /etc/localtime ## to file need configure 3 # - hostPath: # path: /tmp/mylog # type: "" # name: mylog kubectl apply -f 04-log-pilot.yml 5. 配置 kibana 页面 Managenment>index Patterns>Create index pattern #创建日志 Create index pattern> index pattern(tomcat-access*)>Next step #创建时间 Time Filter field name(@timestamp)>Create index pattern #查看日志展示 Discover>tomcat-access* ———————————————— 版权声明:本文为CSDN博主「大虾别跑」的原创文章,遵循CC 4.0 BY-SA版权协议,转载请附上原文出处链接及本声明。 原文链接:https://blog.csdn.net/qq_35583325/article/details/128172276 十、Prometheus 监控 1. 导入离线包 链接:https://pan.baidu.com/s/1DyMJPT8r_TUpI8Dr31SVew?pwd=m1bk 提取码:m1bk #导入上传tar包 sudo docker load -i alertmanager-v0.21.0.tar sudo docker load -i grafana-7.3.4.tar sudo docker load -i k8s-prometheus-adapter-v0.8.2.tar sudo docker load -i kube-rbac-proxy-v0.8.0.tar sudo docker load -i kube-state-metrics-v1.9.7.tar sudo docker load -i node-exporter-v1.0.1.tar sudo docker load -i prometheus-config-reloader-v0.43.2.tar sudo docker load -i prometheus_demo_service.tar sudo docker load -i prometheus-operator-v0.43.2.tar sudo docker load -i prometheus-v2.22.1.tar 2. 主节点创建 #解压下载的代码包 sudo unzip kube-prometheus-master.zip sudo rm -f kube-prometheus-master.zip && cd kube-prometheus-master #这里建议先看下有哪些镜像,便于在下载镜像快的节点上先收集好所有需要的离线docker镜像 find ./ -type f |xargs grep 'image: '|sort|uniq|awk '{print $3}'|grep ^[a-zA-Z]|grep -Evw 'error|kubeRbacProxy'|sort -rn|uniq kubectl create -f manifests/setup kubectl create -f manifests/ #过一会查看创建结果: kubectl -n monitoring get all # 附:清空上面部署的prometheus所有服务: # kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup 3. 访问下 prometheus 的 UI # 修改下prometheus UI的service模式,便于我们访问 # kubectl -n monitoring patch svc prometheus-k8s -p '{"spec":{"type":"NodePort"}}' service/prometheus-k8s patched # kubectl -n monitoring get svc prometheus-k8s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE prometheus-k8s NodePort 10.68.23.79 <none> 9090:22129/TCP 7m43s 3.1 修改用户权限 # kubectl edit clusterrole prometheus-k8s #------ 原始的rules ------- rules: - apiGroups: - "" resources: - nodes/metrics verbs: - get - nonResourceURLs: - /metrics verbs: - get #--------------------------- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: prometheus-k8s rules: - apiGroups: - "" resources: - nodes - services - endpoints - pods - nodes/proxy verbs: - get - list - watch - apiGroups: - "" resources: - configmaps - nodes/metrics verbs: - get - nonResourceURLs: - /metrics verbs: - get 4. 监控 ingress-nginx cat > servicemonitor.yaml <<EOF apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: app: ingress-nginx name: nginx-ingress-scraping namespace: ingress-nginx spec: endpoints: - interval: 30s path: /metrics port: metrics jobLabel: app namespaceSelector: matchNames: - ingress-nginx selector: matchLabels: app: ingress-nginx EOF kubectl apply -f servicemonitor.yaml kubectl -n ingress-nginx get servicemonitors.monitoring.coreos.com 十一、安装 kubesphere3.3 官网参考文档 https://kubesphere.com.cn/docs/v3.3/pluggable-components/alerting/ 1. 部署 kubesphere 时需要默认 StorageClass kubectl edit sc nfs-boge metadata: annotations: storageclass.beta.kubernetes.io/is-default-class: "true" 2. 下载 yaml wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/kubesphere-installer.yaml wget https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/cluster-configuration.yaml #修改cluster-configuration.yaml #将ectd下的 endpointIps改为你的master节点的私有IP地址。 #endpointIps: XX.X.X.X 3. 运行 yaml kubectl apply -f kubesphere-installer.yaml kubectl apply -f cluster-configuration.yaml 4. 查看日志 kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f #访问任意机器的 30880端口 #账号 : admin #密码 : P@88w0rd 5. 解决 etcd 监控证书找不到问题 kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs \ --from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt \ --from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/healthcheck-client.crt \ --from-file=etcd-client.key=/etc/kubernetes/pki/etcd/healthcheck-client.key 6. 在安装后启用告警系统 在 cluster-configuration.yaml 文件中,搜索 alerting,将 enabled 的 false 更改为 true 以启用告警系统。完成后保存文件 alerting: enabled: true # 将“false”更改为“true”。 #运行 kubectl apply -f kubesphere-installer.yaml kubectl apply -f cluster-configuration.yaml 6.0 配置钉钉报警 6.1 钉钉自定义机器配置 添加自定义机器人,安全配置,勾选 ** 加签 ** 6.2 操作步骤 左上角 > 平台管理 > 平台设置 > 通知管理 > 通知配置 > 钉钉 > 群机器人配置 开启 - 已启用 填写自己的 Webhook URL 填写自己的 密钥 (加签) 发送测试信息 确定 查看钉钉群消息。是否发送成功????? 7. 在安装后启用应用商店, 在该 YAML 文件中,搜索 openpitrix,将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置。 openpitrix: store: enabled: true # 将“false”更改为“true”。 #运行 kubectl apply -f kubesphere-installer.yaml kubectl apply -f cluster-configuration.yaml 8. 在安装后启用服务网格 istio 在该配置文件中,搜索 servicemesh,并将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置 servicemesh: enabled: true # 将“false”更改为“true”。 istio: # Customizing the istio installation configuration, refer to https://istio.io/latest/docs/setup/additional-setup/customize-installation/ components: ingressGateways: - name: istio-ingressgateway # 将服务暴露至服务网格之外。默认不开启。 enabled: false cni: enabled: false # 启用后,会在 Kubernetes pod 生命周期的网络设置阶段完成 Istio 网格的 pod 流量转发设置工作。 9. 在安装前启用 DevOps 在该 YAML 文件中,搜索 devops,将 enabled 的 false 改为 true。完成后,点击右下角的确定,保存配置。 devops: enabled: true # 将“false”更改为“true”。 10. 卸载方法 kubectl delete -f cluster-configuration.yaml --force kubectl delete -f kubesphere-installer.yaml --force #删除残余文件 vi del.sh #!/usr/bin/env bash function delete_sure(){ cat << eof $(echo -e "\033[1;36mNote:\033[0m") Delete the KubeSphere cluster, including the module kubesphere-system kubesphere-devops-system kubesphere-devops-worker kubesphere-monitoring-system kubesphere-logging-system openpitrix-system. eof read -p "Please reconfirm that you want to delete the KubeSphere cluster. (yes/no) " ans while [[ "x"$ans != "xyes" && "x"$ans != "xno" ]]; do read -p "Please reconfirm that you want to delete the KubeSphere cluster. (yes/no) " ans done if [[ "x"$ans == "xno" ]]; then exit fi } delete_sure # delete ks-installer kubectl delete deploy ks-installer -n kubesphere-system 2>/dev/null # delete helm for namespaces in kubesphere-system kubesphere-devops-system kubesphere-monitoring-system kubesphere-logging-system openpitrix-system kubesphere-monitoring-federated do helm list -n $namespaces | grep -v NAME | awk '{print $1}' | sort -u | xargs -r -L1 helm uninstall -n $namespaces 2>/dev/null done # delete kubefed kubectl get cc -n kubesphere-system ks-installer -o jsonpath="{.status.multicluster}" | grep enable if [[ $? -eq 0 ]]; then # delete kubefed types resources for kubefed in `kubectl api-resources --namespaced=true --api-group=types.kubefed.io -o name` do kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null done for kubefed in `kubectl api-resources --namespaced=false --api-group=types.kubefed.io -o name` do kubectl delete $kubefed --all 2>/dev/null done # delete kubefed core resouces for kubefed in `kubectl api-resources --namespaced=true --api-group=core.kubefed.io -o name` do kubectl delete -n kube-federation-system $kubefed --all 2>/dev/null done for kubefed in `kubectl api-resources --namespaced=false --api-group=core.kubefed.io -o name` do kubectl delete $kubefed --all 2>/dev/null done # uninstall kubefed chart helm uninstall -n kube-federation-system kubefed 2>/dev/null fi helm uninstall -n kube-system snapshot-controller 2>/dev/null # delete kubesphere deployment & statefulset kubectl delete deployment -n kubesphere-system `kubectl get deployment -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null kubectl delete statefulset -n kubesphere-system `kubectl get statefulset -n kubesphere-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null # delete monitor resources kubectl delete prometheus -n kubesphere-monitoring-system k8s 2>/dev/null kubectl delete Alertmanager -n kubesphere-monitoring-system main 2>/dev/null kubectl delete DaemonSet -n kubesphere-monitoring-system node-exporter 2>/dev/null kubectl delete statefulset -n kubesphere-monitoring-system `kubectl get statefulset -n kubesphere-monitoring-system -o jsonpath="{.items[*].metadata.name}"` 2>/dev/null # delete grafana kubectl delete deployment -n kubesphere-monitoring-system grafana 2>/dev/null kubectl --no-headers=true get pvc -n kubesphere-monitoring-system -o custom-columns=:metadata.namespace,:metadata.name | grep -E kubesphere-monitoring-system | xargs -n2 kubectl delete pvc -n 2>/dev/null # delete pvc pvcs="kubesphere-system|openpitrix-system|kubesphere-devops-system|kubesphere-logging-system" kubectl --no-headers=true get pvc --all-namespaces -o custom-columns=:metadata.namespace,:metadata.name | grep -E $pvcs | xargs -n2 kubectl delete pvc -n 2>/dev/null # delete rolebindings delete_role_bindings() { for rolebinding in `kubectl -n $1 get rolebindings -l iam.kubesphere.io/user-ref -o jsonpath="{.items[*].metadata.name}"` do kubectl -n $1 delete rolebinding $rolebinding 2>/dev/null done } # delete roles delete_roles() { kubectl -n $1 delete role admin 2>/dev/null kubectl -n $1 delete role operator 2>/dev/null kubectl -n $1 delete role viewer 2>/dev/null for role in `kubectl -n $1 get roles -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"` do kubectl -n $1 delete role $role 2>/dev/null done } # remove useless labels and finalizers for ns in `kubectl get ns -o jsonpath="{.items[*].metadata.name}"` do kubectl label ns $ns kubesphere.io/workspace- kubectl label ns $ns kubesphere.io/namespace- kubectl patch ns $ns -p '{"metadata":{"finalizers":null,"ownerReferences":null}}' delete_role_bindings $ns delete_roles $ns done # delete clusterroles delete_cluster_roles() { for role in `kubectl get clusterrole -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"` do kubectl delete clusterrole $role 2>/dev/null done for role in `kubectl get clusterroles | grep "kubesphere" | awk '{print $1}'| paste -sd " "` do kubectl delete clusterrole $role 2>/dev/null done } delete_cluster_roles # delete clusterrolebindings delete_cluster_role_bindings() { for rolebinding in `kubectl get clusterrolebindings -l iam.kubesphere.io/role-template -o jsonpath="{.items[*].metadata.name}"` do kubectl delete clusterrolebindings $rolebinding 2>/dev/null done for rolebinding in `kubectl get clusterrolebindings | grep "kubesphere" | awk '{print $1}'| paste -sd " "` do kubectl delete clusterrolebindings $rolebinding 2>/dev/null done } delete_cluster_role_bindings # delete clusters for cluster in `kubectl get clusters -o jsonpath="{.items[*].metadata.name}"` do kubectl patch cluster $cluster -p '{"metadata":{"finalizers":null}}' --type=merge done kubectl delete clusters --all 2>/dev/null # delete workspaces for ws in `kubectl get workspaces -o jsonpath="{.items[*].metadata.name}"` do kubectl patch workspace $ws -p '{"metadata":{"finalizers":null}}' --type=merge done kubectl delete workspaces --all 2>/dev/null # make DevOps CRs deletable for devops_crd in $(kubectl get crd -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep "devops.kubesphere.io"); do for ns in $(kubectl get ns -ojsonpath='{.items..metadata.name}'); do for devops_res in $(kubectl get $devops_crd -n $ns -oname); do kubectl patch $devops_res -n $ns -p '{"metadata":{"finalizers":[]}}' --type=merge done done done # delete validatingwebhookconfigurations for webhook in ks-events-admission-validate users.iam.kubesphere.io network.kubesphere.io validating-webhook-configuration resourcesquotas.quota.kubesphere.io do kubectl delete validatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null done # delete mutatingwebhookconfigurations for webhook in ks-events-admission-mutate logsidecar-injector-admission-mutate mutating-webhook-configuration do kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io $webhook 2>/dev/null done # delete users for user in `kubectl get users -o jsonpath="{.items[*].metadata.name}"` do kubectl patch user $user -p '{"metadata":{"finalizers":null}}' --type=merge done kubectl delete users --all 2>/dev/null # delete helm resources for resource_type in `echo helmcategories helmapplications helmapplicationversions helmrepos helmreleases`; do for resource_name in `kubectl get ${resource_type}.application.kubesphere.io -o jsonpath="{.items[*].metadata.name}"`; do kubectl patch ${resource_type}.application.kubesphere.io ${resource_name} -p '{"metadata":{"finalizers":null}}' --type=merge done kubectl delete ${resource_type}.application.kubesphere.io --all 2>/dev/null done # delete workspacetemplates for workspacetemplate in `kubectl get workspacetemplates.tenant.kubesphere.io -o jsonpath="{.items[*].metadata.name}"` do kubectl patch workspacetemplates.tenant.kubesphere.io $workspacetemplate -p '{"metadata":{"finalizers":null}}' --type=merge done kubectl delete workspacetemplates.tenant.kubesphere.io --all 2>/dev/null # delete federatednamespaces in namespace kubesphere-monitoring-federated for resource in $(kubectl get federatednamespaces.types.kubefed.io -n kubesphere-monitoring-federated -oname); do kubectl patch "${resource}" -p '{"metadata":{"finalizers":null}}' --type=merge -n kubesphere-monitoring-federated done # delete crds for crd in `kubectl get crds -o jsonpath="{.items[*].metadata.name}"` do if [[ $crd == *kubesphere.io ]] || [[ $crd == *kubefed.io ]] ; then kubectl delete crd $crd 2>/dev/null; fi done # delete relevance ns for ns in kube-federation-system kubesphere-alerting-system kubesphere-controls-system kubesphere-devops-system kubesphere-devops-worker kubesphere-logging-system kubesphere-monitoring-system kubesphere-monitoring-federated openpitrix-system kubesphere-system do kubectl delete ns $ns 2>/dev/null done #执行删除 sh del.sh 十二、 GitLab 安装 1. 单独准备服务器,采用 Docker 安装 docker search gitlab docker pull gitlab/gitlab-ce 2. 准备 docker-compose.yml 文件 mkdir -p /data/git vim /data/git/docker-compose.yml version: '3.1' services: gitlab: image: 'gitlab/gitlab-ce:latest' container_name: gitlab restart: always environment: GITLAB_OMNIBUS_CONFIG: | external_url 'http://10.1.100.225:8929'#自己安装git的服务器IP gitlab_rails['gitlab_shell_ssh_port'] = 2224 ports: - '8929:8929' - '2224:2224' volumes: - './config:/etc/gitlab' - './logs:/var/log/gitlab' - './data:/var/opt/gitlab' 3. 启动容器(需要稍等很久……) cd /data/git docker-compose up -d 4. 访问 GitLab 首页 http://10.1.100.225:8929 5. 查看 root 用户初始密码 docker exec -it gitlab cat /etc/gitlab/initial_root_password 6. 第一次登录网页,需要修改密码 Password ** 右上角 >>**Administrator>Preferences>Password 十三、DevOps 初始化环境 1.linux 系统 安装 Jenkins、jdk 、maven 1. 下载地址 JDK 包下载地址 https://www.oracle.com/java/technologies/downloads/ MAven 下载地址 https://maven.apache.org/download.cgi 2. 安装 jdk maven tar -zxvf jdk-8*.tar.gz -C /usr/local/ tar -zxvf apache-maven-*.tar.gz -C /usr/local/ cd /usr/local mv apache-maven*/ maven mv jdk1.8*/ jdk 2.1 编辑 maven 配置 vim /usr/local/maven/conf/settings.xml <!--#maven配置阿里云仓库,在160行插入--> <mirror> <id>nexus-aliyun</id> <mirrorOf>central</mirrorOf> <name>Nexus aliyun</name> <url>http://maven.aliyun.com/nexus/content/groups/public</url> </mirror> <!--#maven配置jdk,在252行插入--> <profile> <id>jdk1.8</id> <activation> <activeByDefault>true</activeByDefault> <jdk>1.8</jdk> </activation> <properties> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.compilerVersion>1.8</maven.compiler.compilerVersion> </properties> </profile> <!--#maven配置jdk,在257行插入--> <activeProfiles> <activeProfile>jdk1.8</activeProfile> </activeProfiles> 3. 安装 jenkins 3.1 下载 docker pull jenkins/jenkins:2.319.1-lts 3.2 创建 yaml mkdir -p /data/jenkins/ cd /data/jenkins/ vim /data/jenkins/docker-compose.yml version: "3.1" services: jenkins: image: jenkins/jenkins container_name: jenkins ports: - 8080:8080 - 50000:50000 volumes: - ./data/:/var/jenkins_home/ - /var/run/docker.sock:/var/run/docker.sock - /usr/bin/docker:/usr/bin/docker - /etc/docker/daemon.json:/etc/docker/daemon.json 3.3 启动 jenkins #修改Jenkins用户权限 cd /var/run chown root:root docker.sock #其他用户有读和写权限 chmod o+rw docker.sock cd /data/jenkins/ docker-compose up -d #授权 chmod 777 /data/jenkins/data/ cat /data/jenkins/data/hudson.model.UpdateCenter.xml #重新启动Jenkins容器后,由于Jenkins需要下载大量内容,但是由于默认下载地址下载速度较慢, #需要重新设置下载地址为国内镜像站# 清华大学的插件源也可以 # 修改数据卷中的hudson.model.UpdateCenter.xml文件 # 将下载地址替换为http://mirror.esuni.jp/jenkins/updates/update-center.json # 清华大学的插件源也可以 #https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json #重启 docker-compose restart #查看日志 docker logs -f jenkins 3.4 访问页面,安装插件 http://10.1.100.225:8080 1. 输入密码 2. 选择插件来安装 3. 点击安装 4.jenkins 插件安装 中文界面>系统管理>插件管理>可选插件>搜索插件 英文界面> Manage Jenkins–Manage Plugins-Available>搜索插件 Locale Localization Git Parameter Publish Over SSH 5. 配置 jenkins mv /usr/local/maven/ /data/jenkins/data/ mv /usr/local/jdk/ /data/jenkins/data/ 5.1 加载本地 jdk Dashboard > 系统管理 > 全局工具配置 > Add JDK > 去掉对钩 (√)自动安装 NAME jdk8 JAVA_HOME /var/jenkins_home/jdk/ 5.1 加载本地 maven Dashboard > 系统管理 > 全局工具配置 > Add Maven > 去掉对钩 (√)自动安装 NAME maven JAVA_HOME /var/jenkins_home/maven/ Save Apply 保存 应用 运行 mvn 测试 mvn help:system 3.jenkins 拉取测试 系统管理 > 系统配置 > Publish over SSH>SSH Servers>Add #自定义项目名称 name test #主机 IP Hostname 10.1.100.25 #主机用户名 Username root #拉取项目路径 Remote Directory /data/work/mytest 点击高级 √ Use password authentication, or use a different key #输入服务器密码 Passphrase / Password xxxx #点击 测试 Test Configuration Save Apply 保存 应用 4.Jenkins 服务器设置免密登入 k8s-mast 服务器 #Jenkins 服务器 - 进入 jenkins 容器 docker exec -it jenkins bash #进入 jenkins 容器 - 生成免密登录公私钥,根据提示按回车 ssh-keygen -t rsa #进入 jenkins 容器 - 查看 jenkins 秘钥 cat /var/jenkins_home/.ssh/id_rsa.pub #k8s-mast 服务器中 authorized_keys 加入 Jenkins 服务器秘钥 echo “xxxxxx” >> /root/.ssh/authorized_keys 十四、开发环境部署 IDEA 工具下载: 链接:https://pan.baidu.com/s/1Jkyh_kgrT2o388Xiujbdeg?pwd=b7rx 提取码:b7rx 1. windows 配置 maven 和 jdk https://blog.csdn.net/weixin_46565024/article/details/122758111 2. IDEA 简单得项目创建 File>New>Project Spring Initializr>Next Type(选择Maven)>Java Version (选择8) > Next Web> 勾选√Spring Web> Next>Finish (责任编辑:IT) |