在Kubernetes上部署 Redis 三主三从 集群

NFS搭建见： Linux NFS搭建与配置(https://www.iteye.com/blog/maosheng-2517254)

一、创建pv

# cat redis-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv1
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv2
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv3
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv4
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv5
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv6
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv6
    server: 192.101.11.156


# kubectl apply -f redis-pv.yml
persistentvolume/redis-pv1 created
persistentvolume/redis-pv2 created
persistentvolume/redis-pv3 created
persistentvolume/redis-pv4 created
persistentvolume/redis-pv5 created
persistentvolume/redis-pv6 created

# kubectl get pv
NAME           CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                      STORAGECLASS    REASON   AGE
alertmanager   10Gi       RWO            Recycle          Bound       kube-system/alertmanager                            133d
grafana        10Gi       RWO            Recycle          Bound       kube-system/grafana                                 137d
prometheus     10Gi       RWO            Recycle          Bound       kube-system/prometheus                              134d
redis-pv1      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv2      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv3      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv4      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv5      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv6      5Gi        RWO            Recycle          Available                              redis-cluster            10s


二、创建statefulset

# cat redis-sts.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster
data:
  update-node.sh: |
    #!/bin/sh
    REDIS_NODES="/data/nodes.conf"
    sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
    exec "$@"
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file /data/nodes.conf
    cluster-migration-barrier 1
    appendonly yes
    protected-mode no
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster
spec:
  serviceName: redis-cluster
  replicas: 6
  selector:
    matchLabels:
      app: redis-cluster
  template:
    metadata:
      labels:
        app: redis-cluster
    spec:
      containers:
        - name: redis
          image: redis:6.2
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 6379
              name: client
            - containerPort: 16379
              name: gossip
          command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: conf
              mountPath: /conf
              readOnly: false
            - name: data
              mountPath: /data
              readOnly: false
      volumes:
        - name: conf
          configMap:
            name: redis-cluster
            defaultMode: 0755
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 5Gi
        storageClassName: redis-cluster


# kubectl apply -f redis-sts.yml
configmap/redis-cluster created
statefulset.apps/redis-cluster created
You have mail in /var/spool/mail/root

# kubectl get pods -l app=redis-cluster
NAME              READY   STATUS    RESTARTS   AGE
redis-cluster-0   1/1     Running   0          11s
redis-cluster-1   1/1     Running   0          11s
redis-cluster-2   1/1     Running   0          8s
redis-cluster-3   1/1     Running   0          5s
redis-cluster-4   1/1     Running   0          5s
redis-cluster-5   1/1     Running   0          2s

三、创建service

# cat redis-svc.yml
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster
spec:
  externalIPs:
    - 192.101.11.159
    - 192.101.11.160
    - 192.101.11.161
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: 6379
      name: client
    - port: 16379
      targetPort: 16379
      name: gossip
  selector:
    app: redis-cluster


# kubectl apply -f redis-svc.yml
service/redis-cluster created

# kubectl get svc
NAME                TYPE        CLUSTER-IP       EXTERNAL-IP                                    PORT(S)              AGE
redis-cluster       ClusterIP   10.97.104.201    192.101.11.159,192.101.11.160,192.101.11.161   6379/TCP,16379/TCP   8s

四、初始化 Redis Cluster

# kubectl exec -it redis-cluster-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=redis-cluster -o jsonpath='{range.items

}{.status.podIP}:6379 ')

>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.244.1.19:6379 to 10.244.1.17:6379
Adding replica 10.244.2.28:6379 to 10.244.2.26:6379
Adding replica 10.244.2.27:6379 to 10.244.1.18:6379
M: 029cfe88022f45da4dbbee030697326248cefb3c 10.244.1.17:6379
   slots:[0-5460] (5461 slots) master
M: fbd3883ba110068474c80154718cc58bbf36a2c5 10.244.2.26:6379
   slots:[5461-10922] (5462 slots) master
M: 222742c9e2f8e12becb3e6ea4019ae0da563b841 10.244.1.18:6379
   slots:[10923-16383] (5461 slots) master
S: 46a2f4aed697e347b79b4a09bfb719bfd30c57a3 10.244.2.27:6379
   replicates 222742c9e2f8e12becb3e6ea4019ae0da563b841
S: b8f32d48b94436d39f854717c18f3c322f1c3c39 10.244.1.19:6379
   replicates 029cfe88022f45da4dbbee030697326248cefb3c
S: 3a7d20346ace9afa4558b3ff8e41c507c72a9f46 10.244.2.28:6379
   replicates fbd3883ba110068474c80154718cc58bbf36a2c5
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join

>>> Performing Cluster Check (using node 10.244.1.17:6379)
M: 029cfe88022f45da4dbbee030697326248cefb3c 10.244.1.17:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: b8f32d48b94436d39f854717c18f3c322f1c3c39 10.244.1.19:6379
   slots: (0 slots) slave
   replicates 029cfe88022f45da4dbbee030697326248cefb3c
S: 3a7d20346ace9afa4558b3ff8e41c507c72a9f46 10.244.2.28:6379
   slots: (0 slots) slave
   replicates fbd3883ba110068474c80154718cc58bbf36a2c5
M: fbd3883ba110068474c80154718cc58bbf36a2c5 10.244.2.26:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 46a2f4aed697e347b79b4a09bfb719bfd30c57a3 10.244.2.27:6379
   slots: (0 slots) slave
   replicates 222742c9e2f8e12becb3e6ea4019ae0da563b841
M: 222742c9e2f8e12becb3e6ea4019ae0da563b841 10.244.1.18:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

# kubectl exec -it redis-cluster-0 -- redis-cli cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:11
cluster_stats_messages_pong_sent:17
cluster_stats_messages_sent:28
cluster_stats_messages_ping_received:12
cluster_stats_messages_pong_received:11
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:28
[root@hadoop008 redis-cluster]# ^C
[root@hadoop008 redis-cluster]# ^C
You have mail in /var/spool/mail/root


五、验证集群部署

# for x in $(seq 0 5); do echo "redis-cluster-$x"; kubectl exec redis-cluster-$x -- redis-cli role; echo; done
redis-cluster-0
master
518
10.244.1.19
6379
518

redis-cluster-1
master
518
10.244.2.28
6379
518

redis-cluster-2
master
518
10.244.2.27
6379
518

redis-cluster-3
slave
10.244.1.18
6379
connected
518

redis-cluster-4
slave
10.244.1.17
6379
connected
518

redis-cluster-5
slave
10.244.2.26
6379
connected
518