1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
#!/bin/bash
# init
mkdir -p redis-sentinel/{master,slave1,slave2,sentinel1,sentinel2,sentinel3}
mkdir -p redis-sentinel/{master,slave1,slave2}/data
touch redis-sentinel/{master,slave1,slave2}/redis.conf
touch redis-sentinel/{sentinel1,sentinel2,sentinel3}/sentinel.conf
chmod 666 sentinel1/sentinel.conf sentinel2/sentinel.conf sentinel3/sentinel.conf
cd redis-sentinel
# redis config
HOST_IP=10.4.100.125
# master/redis.conf
cat > ./master/redis.conf <<EOF
port 6379
bind 0.0.0.0
protected-mode no
requirepass redis
masterauth redis
appendonly yes
dir /data
EOF
# slave1/redis.conf
cat > ./slave1/redis.conf <<EOF
port 6379
bind 0.0.0.0
protected-mode no
requirepass redis
masterauth redis
replicaof ${HOST_IP} 16379
replica-announce-ip ${HOST_IP}
replica-announce-port 26379
appendonly yes
dir /data
EOF
# slave2/redis.conf
cat > ./slave2/redis.conf <<EOF
port 6379
bind 0.0.0.0
protected-mode no
requirepass redis
masterauth redis
replicaof ${HOST_IP} 16379
replica-announce-ip ${HOST_IP}
replica-announce-port 36379
appendonly yes
dir /data
EOF
# sentinel config
# sentinel1/sentinel.conf
cat > ./sentinel1/sentinel.conf <<EOF
port 26379
sentinel monitor mymaster ${HOST_IP} 16379 2
sentinel auth-pass mymaster redis
sentinel down-after-milliseconds mymaster 5000
sentinel failover-timeout mymaster 60000
sentinel parallel-syncs mymaster 1
sentinel announce-ip ${HOST_IP}
sentinel announce-port 46379
EOF
# sentinel2/sentinel.conf
cat > ./sentinel2/sentinel.conf <<EOF
port 26379
sentinel monitor mymaster ${HOST_IP} 16379 2
sentinel auth-pass mymaster redis
sentinel down-after-milliseconds mymaster 5000
sentinel failover-timeout mymaster 60000
sentinel parallel-syncs mymaster 1
sentinel announce-ip ${HOST_IP}
sentinel announce-port 46479
EOF
# sentinel3/sentinel.conf
cat > ./sentinel3/sentinel.conf <<EOF
port 26379
sentinel monitor mymaster ${HOST_IP} 16379 2
sentinel auth-pass mymaster redis
sentinel down-after-milliseconds mymaster 5000
sentinel failover-timeout mymaster 60000
sentinel parallel-syncs mymaster 1
sentinel announce-ip ${HOST_IP}
sentinel announce-port 46579
EOF
|
1
2
3
4
5
6
7
8
9
10
|
port 6379
bind 0.0.0.0
protected-mode no
requirepass redis
masterauth redis
replicaof <宿主机IP> <宿主机主redis节点映射端口>
replica-announce-ip <宿主机IP>
replica-announce-port <宿主机当前redis节点映射端口>
appendonly yes
dir /data
|
sentinel.conf
1
2
3
4
5
6
7
8
9
10
11
12
|
port 26379
# 客户端不在 docker 网络内 → 关掉 hostname 通告,改回IP模式
# sentinel resolve-hostnames yes
# sentinel announce-hostnames yes
# sentinel monitor mymaster redis-master 6379 2
sentinel monitor mymaster <宿主机IP> <宿主机主redis节点映射端口> 2
sentinel auth-pass mymaster redis
sentinel down-after-milliseconds mymaster 5000
sentinel failover-timeout mymaster 60000
sentinel parallel-syncs mymaster 1
sentinel announce-ip <宿主机IP>
sentinel announce-port <宿主机当前哨兵redis节点映射端口>
|
要点说明:
- 2 是 quorum(3 个哨兵中至少 2 个认为主库挂了才会发起故障转移)
resolve-hostnames yes + announce-hostnames yes:让哨兵用容器服务名而不是 IP 来记录/通告节点,这是 Redis 6.2+ 之后专门为容器场景加的能力,省去手工配置固定 IP 的麻烦
- 重要坑:故障转移发生后,Sentinel 会重写自己的配置文件(把新 master 地址等信息写回去),所以这个
sentinel.conf 文件在容器里必须是可写的,不能用只读挂载
- 这样不管客户端在不在 docker 网络里,只要能路由到这个 IP 段(比如宿主机上,容器网络默认是能从宿主机访问的),就不存在 DNS 解析问题。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
networks:
redis-sentinel:
driver: bridge
services:
redis-master:
image: redis:7.4.10
container_name: redis-master
restart: always
ports:
- "16379:6379"
volumes:
- ./master/redis.conf:/etc/redis/redis.conf
- ./master/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-sentinel
redis-slave1:
image: redis:7.4.10
container_name: redis-slave1
restart: always
ports:
- "26379:6379"
volumes:
- ./slave1/redis.conf:/etc/redis/redis.conf
- ./slave1/data:/data
command: redis-server /etc/redis/redis.conf
depends_on:
- redis-master
networks:
- redis-sentinel
redis-slave2:
image: redis:7.4.10
container_name: redis-slave2
restart: always
ports:
- "36379:6379"
volumes:
- ./slave2/redis.conf:/etc/redis/redis.conf
- ./slave2/data:/data
command: redis-server /etc/redis/redis.conf
depends_on:
- redis-master
networks:
- redis-sentinel
sentinel1:
image: redis:7.4.10
container_name: sentinel1
restart: always
ports:
- "46379:26379"
volumes:
- ./sentinel1/sentinel.conf:/etc/redis/sentinel.conf
command: redis-sentinel /etc/redis/sentinel.conf
depends_on:
- redis-master
- redis-slave1
- redis-slave2
networks:
- redis-sentinel
sentinel2:
image: redis:7.4.10
container_name: sentinel2
restart: always
ports:
- "46479:26379"
volumes:
- ./sentinel2/sentinel.conf:/etc/redis/sentinel.conf
command: redis-sentinel /etc/redis/sentinel.conf
depends_on:
- redis-master
- redis-slave1
- redis-slave2
networks:
- redis-sentinel
sentinel3:
image: redis:7.4.10
container_name: sentinel3
restart: always
ports:
- "46579:26379"
volumes:
- ./sentinel3/sentinel.conf:/etc/redis/sentinel.conf
command: redis-sentinel /etc/redis/sentinel.conf
depends_on:
- redis-master
- redis-slave1
- redis-slave2
networks:
- redis-sentinel
|
1
2
3
4
5
6
7
8
9
10
11
12
13
|
docker-compose up -d
# 查看主从复制状态
docker exec -it redis-master redis-cli -a redis info replication
# 查看哨兵视角
docker exec -it sentinel1 redis-cli -p 26379 sentinel master mymaster
docker exec -it sentinel1 redis-cli -p 26379 sentinel replicas mymaster
docker exec -it sentinel1 redis-cli -p 26379 sentinel sentinels mymaster
# 模拟主库故障,观察自动切换
docker stop redis-master
docker exec -it sentinel2 redis-cli -p 26379 sentinel master mymaster # 几秒后应显示新 master
|
宿主机验证
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
# 从slave1容器内部,测试能不能通过"宿主机IP"连回同一台机器的master端口
$ docker exec -it redis-slave1 redis-cli -h <宿主机IP> -p 6379 -a redis ping
docker exec -it redis-slave1 redis-cli -h 10.4.100.125 -p 6379 -a redis ping
# 应该返回 PONG
# 从sentinel1容器内部同样测试
$ docker exec -it sentinel1 redis-cli -h <宿主机IP> -p 6379 -a redis ping
docker exec -it sentinel1 redis-cli -h 10.4.100.125 -p 6379 -a redis ping
# 1. 主从复制是否正常
docker exec -it redis-master redis-cli -a redis info replication
# 应看到 connected_slaves:2,并且两个slave的ip字段显示的是宿主机IP,不是容器内部IP
# 2. sentinel看到的master地址
docker exec -it sentinel1 redis-cli -h 10.4.100.125 -p 46379 sentinel get-master-addr-by-name mymaster
# 应返回 <宿主机IP> 6379,而不是172.x.x.x这种内部地址
# 3. sentinel看到的从库地址
docker exec -it sentinel1 redis-cli -h 10.4.100.125 -p 46379 sentinel replicas mymaster
# 检查每个slave的ip/port字段,应该也是宿主机IP + 6380/6381
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
#!/bin/bash
# redis-cluster
mkdir -p ./{node1,node2,node3,node4,node5,node6}/data
HOST_IP=10.4.100.125
# node1-6 : port 7001-7006, cluster bus port 17001-17006
for i in {1..6}; do
cat > ./node${i}/redis.conf <<EOF
port 700${i}
bind 0.0.0.0
protected-mode no
requirepass redis
masterauth redis
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
cluster-announce-ip ${HOST_IP}
cluster-announce-port 700${i}
cluster-announce-bus-port 1700${i}
appendonly yes
dir /data
EOF
done
|
关键点:数据端口和总线端口都要 1:1 映射(容器内端口 = 宿主机端口),否则 cluster-announce-port/cluster-announce-bus-port 和实际映射对不上,节点间握手会失败。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
services:
redis-node1:
image: redis:7.4.10
container_name: redis-node1
restart: always
ports:
- "7001:7001"
- "17001:17001"
volumes:
- ./node1/redis.conf:/etc/redis/redis.conf
- ./node1/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
redis-node2:
image: redis:7.4.10
container_name: redis-node2
restart: always
ports:
- "7002:7002"
- "17002:17002"
volumes:
- ./node2/redis.conf:/etc/redis/redis.conf
- ./node2/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
redis-node3:
image: redis:7.4.10
container_name: redis-node3
restart: always
ports:
- "7003:7003"
- "17003:17003"
volumes:
- ./node3/redis.conf:/etc/redis/redis.conf
- ./node3/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
redis-node4:
image: redis:7.4.10
container_name: redis-node4
restart: always
ports:
- "7004:7004"
- "17004:17004"
volumes:
- ./node4/redis.conf:/etc/redis/redis.conf
- ./node4/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
redis-node5:
image: redis:7.4.10
container_name: redis-node5
restart: always
ports:
- "7005:7005"
- "17005:17005"
volumes:
- ./node5/redis.conf:/etc/redis/redis.conf
- ./node5/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
redis-node6:
image: redis:7.4.10
container_name: redis-node6
restart: always
ports:
- "7006:7006"
- "17006:17006"
volumes:
- ./node6/redis.conf:/etc/redis/redis.conf
- ./node6/data:/data
command: redis-server /etc/redis/redis.conf
networks:
- redis-cluster-net
networks:
redis-cluster-net:
driver: bridge
|
1
2
|
docker-compose up -d
docker ps # 确认6个容器都Up
|
因为每个节点要通过"宿主机IP:端口"去连接其他节点(对外宣称的地址就是这个),这条路径要绕一圈回到宿主机再回到另一个容器,不是所有 Docker 环境都默认支持:
bash
1
2
3
|
# 从node1容器内部,测试能不能通过"宿主机IP"连到node2
docker exec -it redis-node1 redis-cli -h <宿主机IP> -p 7002 -a redis ping
# 期望返回 PONG
|
如果返回 PONG:继续下一步创建集群。
如果超时/拒绝连接:说明这台机器的 Docker 网络不支持 hairpin,需要开启内核参数:
1
2
3
4
5
6
7
8
|
# 找到bridge网络对应的网卡(通常是docker0或自定义bridge对应的veth)
docker network inspect redis-cluster_redis-cluster-net | grep -i "com.docker.network.bridge.name"
# 对该网桥开启 route_localnet(如果网桥名是br-xxxxx)
sudo sysctl -w net.ipv4.conf.br-xxxxxxxxxxxx.route_localnet=1
# 或者更通用地对docker0尝试(如果用的是默认bridge)
sudo sysctl -w net.ipv4.conf.docker0.route_localnet=1
|
如果调整内核参数后依然不通,最稳妥的退路是把这一台机器改回 host 网络模式(上一条回复里的方案一),bridge+hairpin 在部分云主机/精简内核环境下确实不总能保证生效。
hairpin 验证通过后:
1
2
3
4
|
docker exec -it redis-node1 redis-cli -a redis --cluster create \
<宿主机IP>:7001 <宿主机IP>:7002 <宿主机IP>:7003 \
<宿主机IP>:7004 <宿主机IP>:7005 <宿主机IP>:7006 \
--cluster-replicas 1
|
会打印类似这样的分配方案,确认无误后输入 yes:
1
2
3
4
5
6
|
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica <IP>:7005 to <IP>:7001
Adding replica <IP>:7006 to <IP>:7002
Adding replica <IP>:7004 to <IP>:7003
|
1
2
3
4
5
6
7
8
9
10
11
|
# 集群状态
docker exec -it redis-node1 redis-cli -a redis -c -h <宿主机IP> -p 7001 CLUSTER INFO
docker exec -it redis-node1 redis-cli -a redis -c -h <宿主机IP> -p 7001 CLUSTER NODES
# 官方巡检
docker exec -it redis-node1 redis-cli -a redis --cluster check <宿主机IP>:7001
# 读写测试(-c 参数让客户端自动跟随MOVED重定向)
docker exec -it redis-node1 redis-cli -a redis -c -h <宿主机IP> -p 7001
> set foo bar
> get foo
|
CLUSTER NODES 里每一行应该显示的都是宿主机IP而不是172.x的内部IP,如果还看到内部IP,说明某个节点的 cluster-announce-ip 没生效,检查对应conf文件是否正确挂载,docker-compose restart 对应节点重新加载。