`
maosheng
  • 浏览: 568336 次
  • 性别: Icon_minigender_1
  • 来自: 北京
社区版块
存档分类
最新评论

在Kubernetes上部署 Redis 三主三从 集群

 
阅读更多
NFS搭建见: Linux NFS搭建与配置(https://www.iteye.com/blog/maosheng-2517254)

一、创建pv

# cat redis-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv1
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv2
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv3
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv4
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv5
    server: 192.101.11.156
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv6
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "redis-cluster"
  nfs:
    path: /data/pv6
    server: 192.101.11.156


# kubectl apply -f redis-pv.yml
persistentvolume/redis-pv1 created
persistentvolume/redis-pv2 created
persistentvolume/redis-pv3 created
persistentvolume/redis-pv4 created
persistentvolume/redis-pv5 created
persistentvolume/redis-pv6 created

# kubectl get pv
NAME           CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM                      STORAGECLASS    REASON   AGE
alertmanager   10Gi       RWO            Recycle          Bound       kube-system/alertmanager                            133d
grafana        10Gi       RWO            Recycle          Bound       kube-system/grafana                                 137d
prometheus     10Gi       RWO            Recycle          Bound       kube-system/prometheus                              134d
redis-pv1      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv2      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv3      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv4      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv5      5Gi        RWO            Recycle          Available                              redis-cluster            10s
redis-pv6      5Gi        RWO            Recycle          Available                              redis-cluster            10s


二、创建statefulset

# cat redis-sts.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-cluster
data:
  update-node.sh: |
    #!/bin/sh
    REDIS_NODES="/data/nodes.conf"
    sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
    exec "$@"
  redis.conf: |+
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file /data/nodes.conf
    cluster-migration-barrier 1
    appendonly yes
    protected-mode no
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-cluster
spec:
  serviceName: redis-cluster
  replicas: 6
  selector:
    matchLabels:
      app: redis-cluster
  template:
    metadata:
      labels:
        app: redis-cluster
    spec:
      containers:
        - name: redis
          image: redis:6.2
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 6379
              name: client
            - containerPort: 16379
              name: gossip
          command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          volumeMounts:
            - name: conf
              mountPath: /conf
              readOnly: false
            - name: data
              mountPath: /data
              readOnly: false
      volumes:
        - name: conf
          configMap:
            name: redis-cluster
            defaultMode: 0755
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 5Gi
        storageClassName: redis-cluster


# kubectl apply -f redis-sts.yml
configmap/redis-cluster created
statefulset.apps/redis-cluster created
You have mail in /var/spool/mail/root

# kubectl get pods -l app=redis-cluster
NAME              READY   STATUS    RESTARTS   AGE
redis-cluster-0   1/1     Running   0          11s
redis-cluster-1   1/1     Running   0          11s
redis-cluster-2   1/1     Running   0          8s
redis-cluster-3   1/1     Running   0          5s
redis-cluster-4   1/1     Running   0          5s
redis-cluster-5   1/1     Running   0          2s

三、创建service

# cat redis-svc.yml
apiVersion: v1
kind: Service
metadata:
  name: redis-cluster
spec:
  externalIPs:
    - 192.101.11.159
    - 192.101.11.160
    - 192.101.11.161
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: 6379
      name: client
    - port: 16379
      targetPort: 16379
      name: gossip
  selector:
    app: redis-cluster


# kubectl apply -f redis-svc.yml
service/redis-cluster created

# kubectl get svc
NAME                TYPE        CLUSTER-IP       EXTERNAL-IP                                    PORT(S)              AGE
redis-cluster       ClusterIP   10.97.104.201    192.101.11.159,192.101.11.160,192.101.11.161   6379/TCP,16379/TCP   8s

四、初始化 Redis Cluster

# kubectl exec -it redis-cluster-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=redis-cluster -o jsonpath='{range.items
  • }{.status.podIP}:6379 ')
  • >>> Performing hash slots allocation on 6 nodes...
    Master[0] -> Slots 0 - 5460
    Master[1] -> Slots 5461 - 10922
    Master[2] -> Slots 10923 - 16383
    Adding replica 10.244.1.19:6379 to 10.244.1.17:6379
    Adding replica 10.244.2.28:6379 to 10.244.2.26:6379
    Adding replica 10.244.2.27:6379 to 10.244.1.18:6379
    M: 029cfe88022f45da4dbbee030697326248cefb3c 10.244.1.17:6379
       slots:[0-5460] (5461 slots) master
    M: fbd3883ba110068474c80154718cc58bbf36a2c5 10.244.2.26:6379
       slots:[5461-10922] (5462 slots) master
    M: 222742c9e2f8e12becb3e6ea4019ae0da563b841 10.244.1.18:6379
       slots:[10923-16383] (5461 slots) master
    S: 46a2f4aed697e347b79b4a09bfb719bfd30c57a3 10.244.2.27:6379
       replicates 222742c9e2f8e12becb3e6ea4019ae0da563b841
    S: b8f32d48b94436d39f854717c18f3c322f1c3c39 10.244.1.19:6379
       replicates 029cfe88022f45da4dbbee030697326248cefb3c
    S: 3a7d20346ace9afa4558b3ff8e41c507c72a9f46 10.244.2.28:6379
       replicates fbd3883ba110068474c80154718cc58bbf36a2c5
    Can I set the above configuration? (type 'yes' to accept): yes
    >>> Nodes configuration updated
    >>> Assign a different config epoch to each node
    >>> Sending CLUSTER MEET messages to join the cluster
    Waiting for the cluster to join

    >>> Performing Cluster Check (using node 10.244.1.17:6379)
    M: 029cfe88022f45da4dbbee030697326248cefb3c 10.244.1.17:6379
       slots:[0-5460] (5461 slots) master
       1 additional replica(s)
    S: b8f32d48b94436d39f854717c18f3c322f1c3c39 10.244.1.19:6379
       slots: (0 slots) slave
       replicates 029cfe88022f45da4dbbee030697326248cefb3c
    S: 3a7d20346ace9afa4558b3ff8e41c507c72a9f46 10.244.2.28:6379
       slots: (0 slots) slave
       replicates fbd3883ba110068474c80154718cc58bbf36a2c5
    M: fbd3883ba110068474c80154718cc58bbf36a2c5 10.244.2.26:6379
       slots:[5461-10922] (5462 slots) master
       1 additional replica(s)
    S: 46a2f4aed697e347b79b4a09bfb719bfd30c57a3 10.244.2.27:6379
       slots: (0 slots) slave
       replicates 222742c9e2f8e12becb3e6ea4019ae0da563b841
    M: 222742c9e2f8e12becb3e6ea4019ae0da563b841 10.244.1.18:6379
       slots:[10923-16383] (5461 slots) master
       1 additional replica(s)
    [OK] All nodes agree about slots configuration.
    >>> Check for open slots...
    >>> Check slots coverage...
    [OK] All 16384 slots covered.

    # kubectl exec -it redis-cluster-0 -- redis-cli cluster info
    cluster_state:ok
    cluster_slots_assigned:16384
    cluster_slots_ok:16384
    cluster_slots_pfail:0
    cluster_slots_fail:0
    cluster_known_nodes:6
    cluster_size:3
    cluster_current_epoch:6
    cluster_my_epoch:1
    cluster_stats_messages_ping_sent:11
    cluster_stats_messages_pong_sent:17
    cluster_stats_messages_sent:28
    cluster_stats_messages_ping_received:12
    cluster_stats_messages_pong_received:11
    cluster_stats_messages_meet_received:5
    cluster_stats_messages_received:28
    [root@hadoop008 redis-cluster]# ^C
    [root@hadoop008 redis-cluster]# ^C
    You have mail in /var/spool/mail/root


    五、验证集群部署

    # for x in $(seq 0 5); do echo "redis-cluster-$x"; kubectl exec redis-cluster-$x -- redis-cli role; echo; done
    redis-cluster-0
    master
    518
    10.244.1.19
    6379
    518

    redis-cluster-1
    master
    518
    10.244.2.28
    6379
    518

    redis-cluster-2
    master
    518
    10.244.2.27
    6379
    518

    redis-cluster-3
    slave
    10.244.1.18
    6379
    connected
    518

    redis-cluster-4
    slave
    10.244.1.17
    6379
    connected
    518

    redis-cluster-5
    slave
    10.244.2.26
    6379
    connected
    518











    分享到:
    评论

    相关推荐

    Global site tag (gtag.js) - Google Analytics