tidb集群基于多副本容灾
·
https://www.bookstack.cn/read/tidb-8.1-zh/4c922925a950e576.md
数据库版本
- tidb8.5.0
集群整体架构
| IP地址 | 系统版本 | 备注 | 区域 |
|---|---|---|---|
| 192.168.222.151 | redhat8.4 | tidb8.5 | 1 |
| 192.168.222.152 | redhat8.4 | tidb8.5 | 1 |
| 192.168.222.153 | redhat8.4 | tidb8.5 | 1 |
| 192.168.222.154 | redhat8.4 | tidb8.5 | 2 |
| 192.168.222.155 | redhat8.4 | tidb8.5 | 3 |
| 192.168.222.156 | redhat8.4 | tidb8.5 | 2 |
| 192.168.222.157 | redhat8.4 | tidb8.5 | 2 |
集群架构介绍
分三个区域，第一个区域 3 个节点，第二个区域 3 个节点，第三个区域 1 个节点
区域 1 作为 primary region，区域 2 作为 secondary region，而区域 3 则作为投票使用的第三个区域
tikv 7 个副本，pd 7 个副本
设置 SSH 免密登录
# - 192.168.222.151 主机上执行集群部署，扩容，缩容等
# 设置密钥
ssh-keygen -t rsa -b 4096
# 分发公钥到所有目标节点
for i in {1..7};do echo $i;ssh-copy-id root@192.168.222.15$i ;done
tidb 插件安装
# 安装 TiUP：
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
# 设置环境变量：
source /root/.bash_profile
# 更新 TiUP：
tiup update --self
tiup update cluster
基于正式环境进行更改及部署
部署 TiDB 集群
- 初始化配置文件：
tiup cluster template > topology.yaml
# 创建新的配置文件
[root@localhost tidb]# cat tidb-cluster.yaml
global:
user: "tidb"
ssh_port: 22
deploy_dir: "/data/tidb/tidbdeploy"
data_dir: "/data/tidb//tidbdata"
pd_servers:
- host: 192.168.222.151
- host: 192.168.222.152
- host: 192.168.222.153
- host: 192.168.222.154
- host: 192.168.222.155
- host: 192.168.222.156
- host: 192.168.222.157
tidb_servers:
- host: 192.168.222.151
- host: 192.168.222.154
tikv_servers:
- host: 192.168.222.151
- host: 192.168.222.152
- host: 192.168.222.153
- host: 192.168.222.154
- host: 192.168.222.155
- host: 192.168.222.156
- host: 192.168.222.157
monitoring_servers:
- host: 192.168.222.153
grafana_servers:
- host: 192.168.222.153
部署集群：
# 查看tidb版本
[root@localhost tidb]# tiup list tidb
# 部署
tiup cluster deploy tidb-cluster v8.5.0 tidb-cluster.yaml -u root
# 卸载集群
# tiup cluster destroy tidb-cluster
启动集群：
tiup cluster start tidb-cluster --init
# tiup cluster start tidb-cluster
查看集群状态：
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: tidb
SSH type: builtin
Dashboard URL: http://192.168.222.152:2379/dashboard
Grafana URL: http://192.168.222.153:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.153:3000 grafana 192.168.222.153 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.151:2379 pd 192.168.222.151 2379/2380 linux/x86_64 Up|L /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Up|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.154:2379 pd 192.168.222.154 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.155:2379 pd 192.168.222.155 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.156:2379 pd 192.168.222.156 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:9090 prometheus 192.168.222.153 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.154:4000 tidb 192.168.222.154 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.151:20160 tikv 192.168.222.151 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.154:20160 tikv 192.168.222.154 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.156:20160 tikv 192.168.222.156 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
Total nodes: 16
### 更改root密码
```shell
/home/data/mysql-8.0.30-el7-x86_64/bin/mysql -h 192.168.222.151 -P 4000 -u root -p'^uD_VmT3J6714@d-H2'
ALTER USER 'root'@'%' IDENTIFIED BY '123456';
tidb 写入部分测试数据
tiup bench tpcc prepare -H 192.168.222.151 -P 4000 -p 123456 -D tpcc --warehouses 1
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+-------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+-------+
| 1 | 192.168.222.153:20160 | 3 | null |
| 4 | 192.168.222.151:20160 | 0 | null |
| 5 | 192.168.222.152:20160 | 29 | null |
| 1001 | 192.168.222.155:20160 | 21 | null |
| 1002 | 192.168.222.154:20160 | 20 | null |
| 12001 | 192.168.222.156:20160 | 20 | null |
+----------+-----------------------+--------------+-------+
编写新的集群文件
[root@localhost tidb]# cat tidb-cluster.yaml
global:
user: "root"
ssh_port: 22
deploy_dir: "/data/tidb/tidbdeploy"
data_dir: "/data/tidb//tidbdata"
server_configs:
tikv:
server.grpc-compression-type: gzip # gzip 启用 TiKV 之间的消息压缩，从而降低网络流量。
pd:
replication.location-labels: ["Region","AZ"] # PD 会根据 TiKV 节点的 Region 和 AZ 配置来进行副本的调度。
pd_servers:
- host: 192.168.222.151
name: "pd-192.168.222.151-2379"
- host: 192.168.222.152
name: "pd-192.168.222.152-2379"
- host: 192.168.222.153
name: "pd-192.168.222.153-2379"
- host: 192.168.222.154
name: "pd-192.168.222.154-2379"
- host: 192.168.222.155
name: "pd-192.168.222.155-2379"
- host: 192.168.222.156
name: "pd-192.168.222.156-2379"
- host: 192.168.222.157
name: "pd-192.168.222.157-2379"
tidb_servers:
- host: 192.168.222.151
- host: 192.168.222.154
tikv_servers: # 在 TiKV 节点中通过 labels 选项来对每个 TiKV 节点所在的 Region 和 AZ 进行标记
- host: 192.168.222.151
config:
server.labels: { Region: "Region1", AZ: "AZ1" }
- host: 192.168.222.152
config:
server.labels: { Region: "Region1", AZ: "AZ2" }
- host: 192.168.222.153
config:
server.labels: { Region: "Region1", AZ: "AZ3" }
- host: 192.168.222.154
config:
server.labels: { Region: "Region2", AZ: "AZ4" }
- host: 192.168.222.156
config:
server.labels: { Region: "Region2", AZ: "AZ6" }
- host: 192.168.222.157
config:
server.labels: { Region: "Region2", AZ: "AZ7" }
- host: 192.168.222.155
config:
server.labels: { Region: "Region3", AZ: "AZ5" }
# 延长区域 3 参加选举的时间，从而避免该区域中的副本被选举为主节点
raftstore.raft-min-election-timeout-ticks: 50
raftstore.raft-max-election-timeout-ticks: 60
monitoring_servers:
- host: 192.168.222.153
grafana_servers:
- host: 192.168.222.153
alertmanager_servers:
- host: 192.168.222.153
按上面的文件更改集群的配置，并重新加载集群
tiup cluster edit-config tidb-cluster
tiup cluster reload tidb-cluster -y
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 12 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4 | 192.168.222.151:20160 | 7 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 5 | 192.168.222.152:20160 | 15 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 1001 | 192.168.222.155:20160 | 13 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 1002 | 192.168.222.154:20160 | 15 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 12001 | 192.168.222.156:20160 | 14 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 33028 | 192.168.222.157:20160 | 6 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
集群的副本数和 Leader 限制进行配置
tiup ctl:v8.5.0 pd config set max-replicas 7
tiup ctl:v8.5.0 pd config set label-property reject-leader Region Region3
# 下面的步骤用于向集群中添加一些测试数据，可选
tiup bench tpcc prepare -H 192.168.222.151 -P 4000 -p 123456 -D tpcc --warehouses 1
指定 PD leader 的优先级
# 优先级数值越大的节点成为 leader 的可能性越高
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.151-2379 6
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.152-2379 5
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.153-2379 4
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.154-2379 3
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.156-2379 2
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.157-2379 1
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.155-2379 0
# 获取成员信息
tiup ctl:v8.5.0 pd member
# PD 实例健康状态
tiup ctl:v8.5.0 pd health
# PD 存储情况
tiup ctl:v8.5.0 pd store
创建 placement rule，并将测试表的主副本固定在区域 1
-- 创建两个 placement rules，第一个是区域 1 作为主区域，在系统正常时使用，第二个是区域 2 作为备区域。
-- 作为主区域，当区域 1 出现问题时，区域 2 会作为主区域。
CREATE PLACEMENT POLICY primary_rule_for_region1 PRIMARY_REGION="Region1" REGIONS="Region1, Region2,Region3";
CREATE PLACEMENT POLICY secondary_rule_for_region2 PRIMARY_REGION="Region2" REGIONS="Region1,Region2,Region3";
-- 将刚刚创建的规则 primary_rule_for_region1 应用到对应的用户表上。
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.district PLACEMENT POLICY=primary_rule_for_region1;
-- 说明：请根据需要修改上面的数据库名称、表名和 placement rule 的名称。
-- 使用类似下面的查询，用户可以查看每个区域包含的 leader 数量，以确认 leader 迁移是否完成。
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 12 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4 | 192.168.222.151:20160 | 7 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 5 | 192.168.222.152:20160 | 15 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 1001 | 192.168.222.155:20160 | 13 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 1002 | 192.168.222.154:20160 | 15 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 12001 | 192.168.222.156:20160 | 14 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 33028 | 192.168.222.157:20160 | 6 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
7 rows in set (0.01 sec)
产生一个sql脚本，将非系统schema中的表的leader都设置到特定的区域上
SET @region_name=primary_rule_for_region1;
SELECT concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') FROM information_schema.tables WHERE table_schema NOT IN ('METRICS_SCHEMA', 'PERFORMANCE_SCHEMA', 'INFORMATION_SCHEMA','mysql');
+------------------------------------------------------------------------------------------------+
| concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') |
+------------------------------------------------------------------------------------------------+
| ALTER TABLE tpcc.warehouse PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.stock PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.orders PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.order_line PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.new_order PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.item PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.history PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.district PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE tpcc.customer PLACEMENT POLICY=primary_rule_for_region1; |
| ALTER TABLE sys.schema_unused_indexes PLACEMENT POLICY=primary_rule_for_region1; |
+------------------------------------------------------------------------------------------------+
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.stock PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.orders PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.order_line PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.new_order PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.item PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.history PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.district PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.customer PLACEMENT POLICY=primary_rule_for_region1;
容灾切换
计划内切换
执行如下命令，将所有用户表和 PD Leader 都切换到区域 2
-- 将之前创建的规则 secondary_rule_for_region2 应用到对应的用户表上。
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=secondary_rule_for_region2;
ALTER TABLE tpcc.district PLACEMENT POLICY=secondary_rule_for_region2;
执行如下命令，调低区域 1 的 PD 节点的优先级，并调高区域 2 的 PD 节点的优先级。
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.151-2379 2
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.152-2379 1
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.153-2379 3
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.154-2379 4
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.156-2379 5
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.157-2379 6
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 10 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4 | 192.168.222.151:20160 | 9 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 5 | 192.168.222.152:20160 | 8 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 1001 | 192.168.222.155:20160 | 13 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 1002 | 192.168.222.154:20160 | 15 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 12001 | 192.168.222.156:20160 | 17 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 33028 | 192.168.222.157:20160 | 10 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
SET @region_name=secondary_rule_for_region2;
SELECT concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') FROM information_schema.tables WHERE table_schema NOT IN ('METRICS_SCHEMA', 'PERFORMANCE_SCHEMA', 'INFORMATION_SCHEMA','mysql');
+------------------------------------------------------------------------------------------------+
| concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') |
+------------------------------------------------------------------------------------------------+
| ALTER TABLE tpcc.warehouse PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.stock PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.orders PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.order_line PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.new_order PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.item PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.history PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.district PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.customer PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE sys.schema_unused_indexes PLACEMENT POLICY=secondary_rule_for_region2; |
+------------------------------------------------------------------------------------------------+
计划外切换
数据记录及验证
mysql> use tpcc
mysql> show tables;
+----------------+
| Tables_in_tpcc |
+----------------+
| customer |
| district |
| history |
| item |
| new_order |
| order_line |
| orders |
| stock |
| warehouse |
+----------------+
mysql> select count(*) from customer;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from district;
+----------+
| count(*) |
+----------+
| 10 |
+----------+
mysql> select count(*) from history;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from item;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
mysql> select count(*) from new_order;
+----------+
| count(*) |
+----------+
| 9000 |
+----------+
mysql> select count(*) from order_line;
+----------+
| count(*) |
+----------+
| 300649 |
+----------+
mysql> select count(*) from orders;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from stock;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
mysql> select count(*) from warehouse;
+----------+
| count(*) |
+----------+
| 1 |
+----------+
类似下面的命令终止区域 1 上所有的 TiKV、TiDB 和 PD 节点
tiup cluster stop tidb-cluster -N 192.168.222.151:20160,192.168.222.152:20160,192.168.222.153:20160,192.168.222.151:2379,192.168.222.152:2379,192.168.222.153:2379
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: root
SSH type: builtin
Dashboard URL: http://192.168.222.154:2379/dashboard
Grafana URL: http://192.168.222.152:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.152:9093 alertmanager 192.168.222.152 9093/9094 linux/x86_64 Up /data/tidb/tidbdata/alertmanager-9093 /data/tidb/tidbdeploy/alertmanager-9093
192.168.222.152:3000 grafana 192.168.222.152 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.151:2379 pd 192.168.222.151 2379/2380 linux/x86_64 Down /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Down /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up|L /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.154:2379 pd 192.168.222.154 2379/2380 linux/x86_64 Up|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.155:2379 pd 192.168.222.155 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:9090 prometheus 192.168.222.152 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.153:4000 tidb 192.168.222.153 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.151:20160 tikv 192.168.222.151 20160/20180 linux/x86_64 Disconnected /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Disconnected /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.154:20160 tikv 192.168.222.154 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
运行类似于下面的命令切换用户表的 leader 到区域 2
-- 将之前创建的规则 secondary_rule_for_region2 应用到对应的用户表上。
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=secondary_rule_for_region2;
ALTER TABLE tpcc.district PLACEMENT POLICY=secondary_rule_for_region2;
---可以使用类似下面的查询查看每个区域包含的 leader 数量，以确认 leader 迁移是否完成。
SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 7 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region2"}] |
| 4 | 192.168.222.154:20160 | 16 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 7 | 192.168.222.152:20160 | 0 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 8 | 192.168.222.155:20160 | 1 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 9 | 192.168.222.151:20160 | 0 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
SET @region_name=secondary_rule_for_region2;
SELECT concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') FROM information_schema.tables WHERE table_schema NOT IN ('METRICS_SCHEMA', 'PERFORMANCE_SCHEMA', 'INFORMATION_SCHEMA','mysql');
+------------------------------------------------------------------------------------------------+
| concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', @region_name, ';') |
+------------------------------------------------------------------------------------------------+
| ALTER TABLE tpcc.warehouse PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.stock PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.orders PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.order_line PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.new_order PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.item PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.history PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.district PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE tpcc.customer PLACEMENT POLICY=secondary_rule_for_region2; |
| ALTER TABLE sys.schema_unused_indexes PLACEMENT POLICY=secondary_rule_for_region2; |
+------------------------------------------------------------------------------------------------+
数据记录及验证
mysql> use tpcc
mysql> show tables;
+----------------+
| Tables_in_tpcc |
+----------------+
| customer |
| district |
| history |
| item |
| new_order |
| order_line |
| orders |
| stock |
| warehouse |
+----------------+
mysql> select count(*) from customer;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from district;
+----------+
| count(*) |
+----------+
| 10 |
+----------+
mysql> select count(*) from history;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from item;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
mysql> select count(*) from new_order;
+----------+
| count(*) |
+----------+
| 9000 |
+----------+
mysql> select count(*) from order_line;
+----------+
| count(*) |
+----------+
| 300649 |
+----------+
mysql> select count(*) from orders;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
mysql> select count(*) from stock;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
mysql> select count(*) from warehouse;
+----------+
| count(*) |
+----------+
| 1 |
+----------+
mysql> select * from stock limit 2\G;
*************************** 1. row ***************************
s_i_id: 1
s_w_id: 1
s_quantity: 19
s_dist_01: UKIQNGEPUWTSYGZMCDQKKYJP
s_dist_02: CHIXATXOZRCSWBOBQRQWSHRO
s_dist_03: YBQZDPJBBXPQBXMOSZDDEBQB
s_dist_04: CDDAJXSODFJHEAYKZVJPJEAB
s_dist_05: VWWSQAJKPWFAGXOJJKBBGYOQ
s_dist_06: IYBYBREIKWGBUGCDPPWFRTNJ
s_dist_07: CYKQNODRFMOSQSMMLUFGZUFT
s_dist_08: NJHTWYFHNSBCCEOYCXKMWNKP
s_dist_09: YCLAGDLLJQOMDHCITZFQLEIV
s_dist_10: HLCVXIONMBDXLQYVGWBRBYJP
s_ytd: 0
s_order_cnt: 0
s_remote_cnt: 0
s_data: Gj6oRKDsiPuGNGaJk7XHr7Ts06JPfRYj5I32yFotmrNCY
*************************** 2. row ***************************
s_i_id: 2
s_w_id: 1
s_quantity: 87
s_dist_01: ITEWHXFVJGIPGNTQKWFLRQGL
s_dist_02: JYEOCWWFSJTISVPDKJTBEWRS
s_dist_03: ZFFUPBNESPMUJZYQSGCMWORJ
s_dist_04: CGNQASRMKBVBECHTUUEEIRWF
s_dist_05: TJIHTLDJTFNSQPNUJTCUOEQO
s_dist_06: UFVRQMXCQSZFZROZLQYBASDS
s_dist_07: RBYXTILQKJCJOCNLALQSATYB
s_dist_08: FQKTRFNOTPPEBLKLSMLAJPRZ
s_dist_09: MEOWFUZGTWVZPJTXWZOHFDIB
s_dist_10: XHVAROWLRCTSLIFGGBANLUHV
s_ytd: 0
s_order_cnt: 0
s_remote_cnt: 0
s_data: rqWUJSVNqGMSVWB0u5qn9zNgXyOn8nXQI7zw3KT
2 rows in set (0.01 sec)
扩容
# 新增两台机器 192.168.222.158，192.168.222.159
# 编写配置文件
[root@localhost tidb]# cat scale-tidb-cluster.yaml
global:
user: "root"
ssh_port: 22
deploy_dir: "/data/tidb/tidbdeploy"
data_dir: "/data/tidb//tidbdata"
server_configs:
tikv:
server.grpc-compression-type: gzip # gzip 启用 TiKV 之间的消息压缩，从而降低网络流量。
pd:
replication.location-labels: ["Region","AZ"] # PD 会根据 TiKV 节点的 Region 和 AZ 配置来进行副本的调度。
pd_servers:
- host: 192.168.222.158
name: "pd-192.168.222.158-2379"
- host: 192.168.222.159
name: "pd-192.168.222.159-2379"
tikv_servers: # 在 TiKV 节点中通过 labels 选项来对每个 TiKV 节点所在的 Region 和 AZ 进行标记
- host: 192.168.222.158
config:
server.labels: { Region: "Region1", AZ: "AZ8" }
- host: 192.168.222.159
config:
server.labels: { Region: "Region2", AZ: "AZ9" }
# tiup cluster scale-out tidb-cluster scale-tidb-cluster.yaml -u root
查看集群的情况
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: root
SSH type: builtin
Dashboard URL: http://192.168.222.154:2379/dashboard
Dashboard URLs: http://192.168.222.154:2379/dashboard
Grafana URL: http://192.168.222.153:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.153:9093 alertmanager 192.168.222.153 9093/9094 linux/x86_64 Up /data/tidb/tidbdata/alertmanager-9093 /data/tidb/tidbdeploy/alertmanager-9093
192.168.222.153:3000 grafana 192.168.222.153 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.151:2379 pd 192.168.222.151 2379/2380 linux/x86_64 Up|L /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.154:2379 pd 192.168.222.154 2379/2380 linux/x86_64 Up|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.155:2379 pd 192.168.222.155 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.156:2379 pd 192.168.222.156 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.157:2379 pd 192.168.222.157 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.158:2379 pd 192.168.222.158 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.159:2379 pd 192.168.222.159 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:9090 prometheus 192.168.222.153 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.154:4000 tidb 192.168.222.154 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.151:20160 tikv 192.168.222.151 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.154:20160 tikv 192.168.222.154 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.156:20160 tikv 192.168.222.156 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.157:20160 tikv 192.168.222.157 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.158:20160 tikv 192.168.222.158 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.159:20160 tikv 192.168.222.159 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
Total nodes: 23
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.155:20160 | 12 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 2 | 192.168.222.152:20160 | 18 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 6 | 192.168.222.156:20160 | 0 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 8 | 192.168.222.157:20160 | 9 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 10 | 192.168.222.151:20160 | 2 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 12 | 192.168.222.153:20160 | 12 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 13 | 192.168.222.154:20160 | 11 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 4005 | 192.168.222.158:20160 | 9 | [{"key": "AZ", "value": "AZ8"}, {"key": "Region", "value": "Region1"}] |
| 4006 | 192.168.222.159:20160 | 9 | [{"key": "AZ", "value": "AZ9"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
重新对集群进行设置
## 集群的副本数和 Leader 限制进行配置
tiup ctl:v8.5.0 pd config set max-replicas 9
tiup ctl:v8.5.0 pd config set label-property reject-leader Region Region3
## 指定 PD leader 的优先级
# 优先级数值越大的节点成为 leader 的可能性越高
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.158-2379 8
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.151-2379 7
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.152-2379 6
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.153-2379 5
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.154-2379 4
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.156-2379 3
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.157-2379 2
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.159-2379 1
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.155-2379 0
# 获取成员信息
tiup ctl:v8.5.0 pd member
# PD 实例健康状态
tiup ctl:v8.5.0 pd health
# PD 存储情况
tiup ctl:v8.5.0 pd store
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.155:20160 | 10 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 2 | 192.168.222.152:20160 | 13 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 6 | 192.168.222.156:20160 | 6 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 8 | 192.168.222.157:20160 | 9 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 10 | 192.168.222.151:20160 | 3 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 12 | 192.168.222.153:20160 | 12 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 13 | 192.168.222.154:20160 | 11 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 4005 | 192.168.222.158:20160 | 9 | [{"key": "AZ", "value": "AZ8"}, {"key": "Region", "value": "Region1"}] |
| 4006 | 192.168.222.159:20160 | 9 | [{"key": "AZ", "value": "AZ9"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
缩容
重新对集群进行设置
## 集群的副本数和 Leader 限制进行配置
tiup ctl:v8.5.0 pd config set max-replicas 7
惊醒所容
# 先缩容 tikv，再缩容 pd
tiup cluster scale-in tidb-cluster --node=192.168.222.151:20160
tiup cluster scale-in tidb-cluster --node=192.168.222.154:20160
tiup cluster scale-in tidb-cluster --node=192.168.222.151:2379
tiup cluster scale-in tidb-cluster --node=192.168.222.154:2379
查看集群的情况
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: root
SSH type: builtin
Dashboard URL: http://192.168.222.155:2379/dashboard
Dashboard URLs: http://192.168.222.155:2379/dashboard
Grafana URL: http://192.168.222.153:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.153:9093 alertmanager 192.168.222.153 9093/9094 linux/x86_64 Up /data/tidb/tidbdata/alertmanager-9093 /data/tidb/tidbdeploy/alertmanager-9093
192.168.222.153:3000 grafana 192.168.222.153 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.155:2379 pd 192.168.222.155 2379/2380 linux/x86_64 Up|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.156:2379 pd 192.168.222.156 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.157:2379 pd 192.168.222.157 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.158:2379 pd 192.168.222.158 2379/2380 linux/x86_64 Up|L /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.159:2379 pd 192.168.222.159 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:9090 prometheus 192.168.222.153 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.154:4000 tidb 192.168.222.154 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.156:20160 tikv 192.168.222.156 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.157:20160 tikv 192.168.222.157 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.158:20160 tikv 192.168.222.158 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.159:20160 tikv 192.168.222.159 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
Total nodes: 19
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.155:20160 | 10 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
| 2 | 192.168.222.152:20160 | 16 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 6 | 192.168.222.156:20160 | 10 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 8 | 192.168.222.157:20160 | 13 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 12 | 192.168.222.153:20160 | 12 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4005 | 192.168.222.158:20160 | 10 | [{"key": "AZ", "value": "AZ8"}, {"key": "Region", "value": "Region1"}] |
| 4006 | 192.168.222.159:20160 | 11 | [{"key": "AZ", "value": "AZ9"}, {"key": "Region", "value": "Region2"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
只保留一个 Region1 副本，其他副本全部清除
重新构建的 多副本 集群来进行的测试
搭建多副本集群的关键命令回顾
不用进行操作，只是做为命令回退的依据
## 集群的副本数和 Leader 限制进行配置
tiup ctl:v8.5.0 pd config set max-replicas 7
tiup ctl:v8.5.0 pd config set label-property reject-leader Region Region3
## 指定 PD leader 的优先级
# 优先级数值越大的节点成为 leader 的可能性越高
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.151-2379 6
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.152-2379 5
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.153-2379 4
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.154-2379 3
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.156-2379 2
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.157-2379 1
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.155-2379 0
## 创建 placement rule，并将测试表的主副本固定在区域 1
-- 创建两个 placement rules，第一个是区域 1 作为主区域，在系统正常时使用，第二个是区域 2 作为备区域。
-- 作为主区域，当区域 1 出现问题时，区域 2 会作为主区域。
CREATE PLACEMENT POLICY primary_rule_for_region1 PRIMARY_REGION="Region1" REGIONS="Region1, Region2,Region3";
CREATE PLACEMENT POLICY secondary_rule_for_region2 PRIMARY_REGION="Region2" REGIONS="Region1,Region2,Region3";
-- 将刚刚创建的规则 primary_rule_for_region1 应用到对应的用户表上。
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.stock PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.orders PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.order_line PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.new_order PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.item PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.history PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.district PLACEMENT POLICY=primary_rule_for_region1;
ALTER TABLE tpcc.customer PLACEMENT POLICY=primary_rule_for_region1;
进行多副本命令设置回退
## 查询并将将表的副本策略恢复为默认设置
mysql> SELECT concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', 'DEFAULT', ';') FROM information_schema.tables WHERE table_schema NOT IN ('METRICS_SCHEMA', 'PERFORMANCE_SCHEMA', 'INFORMATION_SCHEMA','mysql');
+---------------------------------------------------------------------------------------------+
| concat('ALTER TABLE ', table_schema, '.', table_name, ' PLACEMENT POLICY=', 'DEFAULT', ';') |
+---------------------------------------------------------------------------------------------+
| ALTER TABLE tpcc.warehouse PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.stock PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.orders PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.order_line PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.new_order PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.item PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.history PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.district PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE tpcc.customer PLACEMENT POLICY=DEFAULT; |
| ALTER TABLE sys.schema_unused_indexes PLACEMENT POLICY=DEFAULT; |
+---------------------------------------------------------------------------------------------+
## 更改
ALTER TABLE tpcc.warehouse PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.stock PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.orders PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.order_line PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.new_order PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.item PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.history PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.district PLACEMENT POLICY=DEFAULT;
ALTER TABLE tpcc.customer PLACEMENT POLICY=DEFAULT;
## 删除表的副本策略
DROP PLACEMENT POLICY primary_rule_for_region1;
DROP PLACEMENT POLICY secondary_rule_for_region2;
## 清除 Leader 优先级设置
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.151-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.152-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.153-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.154-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.156-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.157-2379 0
tiup ctl:v8.5.0 pd member leader_priority pd-192.168.222.155-2379 0
## 撤销拒绝 Leader 的标签属性
tiup ctl:v8.5.0 pd config set label-property reject-leader Region ""
## 设置最大副本数为 1
tiup ctl:v8.5.0 pd config set max-replicas 1
查看 leader 信息
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 8 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4 | 192.168.222.152:20160 | 15 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 7 | 192.168.222.157:20160 | 14 | [{"key": "AZ", "value": "AZ7"}, {"key": "Region", "value": "Region2"}] |
| 8 | 192.168.222.151:20160 | 10 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
| 9 | 192.168.222.154:20160 | 10 | [{"key": "AZ", "value": "AZ4"}, {"key": "Region", "value": "Region2"}] |
| 10 | 192.168.222.156:20160 | 10 | [{"key": "AZ", "value": "AZ6"}, {"key": "Region", "value": "Region2"}] |
| 11 | 192.168.222.155:20160 | 15 | [{"key": "AZ", "value": "AZ5"}, {"key": "Region", "value": "Region3"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
查看集群信息
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: root
SSH type: builtin
Dashboard URL: http://192.168.222.154:2379/dashboard
Dashboard URLs: http://192.168.222.154:2379/dashboard
Grafana URL: http://192.168.222.153:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.153:9093 alertmanager 192.168.222.153 9093/9094 linux/x86_64 Up /data/tidb/tidbdata/alertmanager-9093 /data/tidb/tidbdeploy/alertmanager-9093
192.168.222.153:3000 grafana 192.168.222.153 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.151:2379 pd 192.168.222.151 2379/2380 linux/x86_64 Up|L /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.154:2379 pd 192.168.222.154 2379/2380 linux/x86_64 Up|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.155:2379 pd 192.168.222.155 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.156:2379 pd 192.168.222.156 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.157:2379 pd 192.168.222.157 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:9090 prometheus 192.168.222.153 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.154:4000 tidb 192.168.222.154 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.151:20160 tikv 192.168.222.151 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.154:20160 tikv 192.168.222.154 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.156:20160 tikv 192.168.222.156 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.157:20160 tikv 192.168.222.157 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
Total nodes: 19
进行缩容
tiup cluster scale-in tidb-cluster --node=192.168.222.154:20160
tiup cluster scale-in tidb-cluster --node=192.168.222.157:20160
tiup cluster scale-in tidb-cluster --node=192.168.222.156:20160
tiup cluster scale-in tidb-cluster --node=192.168.222.155:20160
# 进行pd的缩容
tiup cluster scale-in tidb-cluster --node=192.168.222.154:2379
tiup cluster scale-in tidb-cluster --node=192.168.222.155:2379
tiup cluster scale-in tidb-cluster --node=192.168.222.156:2379
tiup cluster scale-in tidb-cluster --node=192.168.222.157:2379
# 进行tidb缩容
tiup cluster scale-in tidb-cluster --node=192.168.222.154:4000
查看集群的情况
[root@localhost tidb]# sh -x status-tidb.sh
+ /root/.tiup/bin/tiup cluster display tidb-cluster
Cluster type: tidb
Cluster name: tidb-cluster
Cluster version: v8.5.0
Deploy user: root
SSH type: builtin
Dashboard URL: http://192.168.222.151:2379/dashboard
Dashboard URLs: http://192.168.222.151:2379/dashboard
Grafana URL: http://192.168.222.153:3000
ID Role Host Ports OS/Arch Status Data Dir Deploy Dir
-- ---- ---- ----- ------- ------ -------- ----------
192.168.222.153:9093 alertmanager 192.168.222.153 9093/9094 linux/x86_64 Up /data/tidb/tidbdata/alertmanager-9093 /data/tidb/tidbdeploy/alertmanager-9093
192.168.222.153:3000 grafana 192.168.222.153 3000 linux/x86_64 Up - /data/tidb/tidbdeploy/grafana-3000
192.168.222.151:2379 pd 192.168.222.151 2379/2380 linux/x86_64 Up|L|UI /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.152:2379 pd 192.168.222.152 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:2379 pd 192.168.222.153 2379/2380 linux/x86_64 Up /data/tidb/tidbdata/pd-2379 /data/tidb/tidbdeploy/pd-2379
192.168.222.153:9090 prometheus 192.168.222.153 9090/12020 linux/x86_64 Up /data/tidb/tidbdata/prometheus-9090 /data/tidb/tidbdeploy/prometheus-9090
192.168.222.151:4000 tidb 192.168.222.151 4000/10080 linux/x86_64 Up - /data/tidb/tidbdeploy/tidb-4000
192.168.222.151:20160 tikv 192.168.222.151 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.152:20160 tikv 192.168.222.152 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.153:20160 tikv 192.168.222.153 20160/20180 linux/x86_64 Up /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.154:20160 tikv 192.168.222.154 20160/20180 linux/x86_64 Tombstone /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.155:20160 tikv 192.168.222.155 20160/20180 linux/x86_64 Tombstone /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.156:20160 tikv 192.168.222.156 20160/20180 linux/x86_64 Tombstone /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
192.168.222.157:20160 tikv 192.168.222.157 20160/20180 linux/x86_64 Tombstone /data/tidb/tidbdata/tikv-20160 /data/tidb/tidbdeploy/tikv-20160
Total nodes: 14
There are some nodes can be pruned:
Nodes: [192.168.222.154:20160 192.168.222.156:20160 192.168.222.157:20160 192.168.222.155:20160]
You can destroy them with the command: `tiup cluster prune tidb-cluster`
副本情况
mysql> SELECT STORE_ID, address, leader_count, label FROM INFORMATION_SCHEMA.TIKV_STORE_STATUS ORDER BY store_id;
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| STORE_ID | address | leader_count | label |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
| 1 | 192.168.222.153:20160 | 0 | [{"key": "AZ", "value": "AZ3"}, {"key": "Region", "value": "Region1"}] |
| 4 | 192.168.222.152:20160 | 5 | [{"key": "AZ", "value": "AZ2"}, {"key": "Region", "value": "Region1"}] |
| 8 | 192.168.222.151:20160 | 0 | [{"key": "AZ", "value": "AZ1"}, {"key": "Region", "value": "Region1"}] |
+----------+-----------------------+--------------+------------------------------------------------------------------------+
3 rows in set (0.00 sec)
查看tidb数据
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| INFORMATION_SCHEMA |
| METRICS_SCHEMA |
| PERFORMANCE_SCHEMA |
| mysql |
| sys |
| test |
| tpcc |
+--------------------+
7 rows in set (0.00 sec)
mysql> use tpcc
Database changed
mysql> show tables;
+----------------+
| Tables_in_tpcc |
+----------------+
| customer |
| district |
| history |
| item |
| new_order |
| order_line |
| orders |
| stock |
| warehouse |
+----------------+
9 rows in set (0.00 sec)
mysql> select count(*) from customer;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
1 row in set (0.05 sec)
mysql> select count(*) from district;
+----------+
| count(*) |
+----------+
| 10 |
+----------+
1 row in set (0.00 sec)
mysql> select count(*) from history;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
1 row in set (0.02 sec)
mysql> select count(*) from item;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
1 row in set (0.09 sec)
mysql> select count(*) from new_order;
+----------+
| count(*) |
+----------+
| 9000 |
+----------+
1 row in set (0.01 sec)
mysql> select count(*) from order_line;
+----------+
| count(*) |
+----------+
| 300668 |
+----------+
1 row in set (0.21 sec)
mysql> select count(*) from orders;
+----------+
| count(*) |
+----------+
| 30000 |
+----------+
1 row in set (0.03 sec)
mysql> select count(*) from stock;
+----------+
| count(*) |
+----------+
| 100000 |
+----------+
1 row in set (0.06 sec)
mysql> select count(*) from warehouse;
+----------+
| count(*) |
+----------+
| 1 |
+----------+
1 row in set (0.01 sec)
汇聚全球AI编程工具，助力开发者即刻编程。
更多推荐
4
0- 0
已为社区贡献1条内容
所有评论(0)