容器国产化适配

龙芯k8s适配

Posted by 爱折腾的工程师 on Saturday, April 25, 2020

环境

  • cpu架构:mips64

  • 操作系统: CentOS 7

  • kernel版本:3.10.0-693.lns7.8.mips64el

  • docker: 18.06.3.ce

  • kubernetes: 1.15.9-0

部署准备

配置静态hosts解析

  1. 禁用selinux,防火墙,swap

    [root@compute ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
    [root@compute ~]# setenforce 0
    
    [root@compute ~]# getenforce 
    Disabled
    

    关闭firewalld

    [root@compute ~]# systemctl status firewalld.service 
    ● firewalld.service - firewalld - dynamic firewall daemon
       Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
       Active: inactive (dead)
         Docs: man:firewalld(1)
    

    临时关闭swag

    [root@master ~]# swapoff -a  
    

    编辑/etc/fstab,禁用swap分区的挂载

    [root@compute ~]# vim /etc/fstab
    #UUID=c5b3070e-4f41-4282-bf1c-359b90ebfbb7 swap                    swap    defaults        0 0
    

    swappiness参数调整

    [root@master ~]# vim /etc/sysctl.d/k8s.conf
    vm.swappiness=0
    [root@master ~]# sysctl -p /etc/sysctl.d/k8s.conf
    
  2. master和node节点安装以下软件包

    下载地址:http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/

    # wget -c http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubectl-1.15.9-0.mips64el.rpm \
        http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubernetes-cni-0.7.5-0.mips64el.rpm \
        http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubelet-1.15.9-0.mips64el.rpm \
        http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubeadm-1.15.9-0.mips64el.rpm
    
    # rpm -ivh kubeadm-1.15.9-0.mips64el.rpm \
        kubectl-1.15.9-0.mips64el.rpm \
        kubelet-1.15.9-0.mips64el.rpm \
        kubernetes-cni-0.7.5-0.mips64el.rpm 
    
  3. 所有节点安装docker

    # yum install -y docker-ce socat
    
    # vim /usr/lib/systemd/system/docker.service 
    [Unit]
    Description=Docker Application Container Engine
    Documentation=https://docs.docker.com
    After=network-online.target firewalld.service
    Wants=network-online.target
    
    [Service]
    Type=notify
    EnvironmentFile=-/etc/sysconfig/docker
    ExecStart=/usr/bin/dockerd
    ExecReload=/bin/kill -s HUP $MAINPID
    LimitNOFILE=1048576
    LimitNPROC=infinity
    LimitCORE=infinity
    TimeoutStartSec=0
    Delegate=yes
    KillMode=process
    Restart=always
    StartLimitBurst=3
    StartLimitInterval=60s
    
    [Install]
    WantedBy=multi-user.target
    
    # systemctl daemon-reload 
    # systemctl start docker
    # systemctl enable docker.service
    
    # systemctl enable kubelet.service
    

部署

kubeadm init --pod-network-cidr=10.244.0.0/16 \
    --apiserver-advertise-address=<你的apiserver-ip> \
    --kubernetes-version=v1.15.9 \
    --image-repository docker.io/loongnixk8s

只有一个节点,去除污点,让master也可以跑负载

kubectl taint node compute node-role.kubernetes.io/master-

安装flannel

# mkdir -p /etc/cni/net.d/
[root@compute ~]# vim kube-flannel.yml 
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      hostNetwork: true
      nodeSelector:
        beta.kubernetes.io/arch: mips64le
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: loongnixk8s/flannel-mips64le:0.10.0 
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: loongnixk8s/flannel-mips64le:0.10.0 
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: true
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
[root@compute ~]# kubectl apply -f kube-flannel.yml 
[root@compute ~]# kubectl get nodes
NAME      STATUS   ROLES    AGE   VERSION
compute   Ready    master   14m   v1.15.9

测试

# kubectl run nginx --image=loongnixk8s/nginx:1.17.7 --replicas=1
[root@compute ~]# kubectl get po
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7c4577b99b-qsl4b   1/1     Running   0          77s
[root@compute ~]# kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
service/nginx exposed
[root@compute ~]# kubectl get svc nginx
NAME    TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
nginx   NodePort   10.97.240.70   <none>        88:30296/TCP   5s
[root@compute ~]# curl http://<your-apiserver-ip>:30296/
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

参考链接

「真诚赞赏,手留余香」

爱折腾的工程师

真诚赞赏,手留余香

使用微信扫描二维码完成支付