node 1 ===================================================================================================================================== -bash-4.1$ mysql -usstuser -psstpass --execute="SHOW GLOBAL STATUS where variable_name in ('wsrep_cluster_size','wsrep_cluster_status','wsrep_incoming_addresses','wsrep_connected','wsrep_ready','wsrep_local_state_comment','wsrep_cluster_conf_id');" mysql: [Warning] Using a password on the command line interface can be insecure. ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/lib/mysql/mysql.sock' (2) -bash-4.1$ ls -l /mysql/mysql.sock srwxrwxrwx. 1 mysql mysql 0 Dec 21 10:03 /mysql/mysql.sock -bash-4.1$ clustercheck HTTP/1.1 503 Service Unavailable Content-Type: text/plain Connection: close Content-Length: 57 Percona XtraDB Cluster Node is not synced or non-PRIM. [root@pld1fmk0113 ~]# netstat -tlpn Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 127.0.0.1:199 0.0.0.0:* LISTEN 2793/snmpd tcp 0 0 0.0.0.0:42924 0.0.0.0:* LISTEN 2476/rpc.statd tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1871/rpcbind tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 2810/sshd tcp 0 0 0.0.0.0:4567 0.0.0.0:* LISTEN 64636/mysqld tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN 2593/cupsd tcp 0 0 :::5989 :::* LISTEN 3074/cimserver tcp 0 0 :::3306 :::* LISTEN 64636/mysqld tcp 0 0 :::111 :::* LISTEN 1871/rpcbind tcp 0 0 :::35762 :::* LISTEN 2476/rpc.statd tcp 0 0 :::22 :::* LISTEN 2810/sshd tcp 0 0 :::23 :::* LISTEN 109036/xinetd tcp 0 0 ::1:631 :::* LISTEN 2593/cupsd [root@pld1fmk0113 ~]# cat /etc/sysconfig/iptables # Generated by iptables-save v1.4.7 on Mon Jan 7 10:25:24 2019 *filter :INPUT ACCEPT [26:1928] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [8:746] -A INPUT -p tcp -m tcp --dport 4444 -j ACCEPT -A INPUT -p tcp -m tcp --dport 4567 -j ACCEPT -A INPUT -p tcp -m tcp --dport 4568 -j ACCEPT COMMIT # Completed on Mon Jan 7 10:25:24 2019 [root@pld1fmk0113 ~]# telnet 172.19.101.108 4444 Trying 172.19.101.108... telnet: connect to address 172.19.101.108: Connection refused telnet: Unable to connect to remote host: Connection refused [root@pld1fmk0113 ~]# telnet 172.19.101.108 4567 Trying 172.19.101.108... telnet: connect to address 172.19.101.108: Connection refused telnet: Unable to connect to remote host: Connection refused [root@pld1fmk0113 ~]# telnet 172.19.101.108 4568 Trying 172.19.101.108... telnet: connect to address 172.19.101.108: Connection refused telnet: Unable to connect to remote host: Connection refused [root@pld1fmk0113 ~]# ===================================================================================================================================== /mysql/log/mysqld.log 2019-01-07T06:37:35.894091Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') connection established to b832ebdd tcp://172.19.101.108:4567 2019-01-07T06:37:35.894510Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 2019-01-07T06:37:36.392394Z 0 [Note] WSREP: declaring b832ebdd at tcp://172.19.101.108:4567 stable 2019-01-07T06:37:36.392982Z 0 [Note] WSREP: Node 9cca12e4 state primary 2019-01-07T06:37:36.393307Z 0 [Note] WSREP: Current view of cluster as seen by this node view (view_id(PRIM,9cca12e4,36) memb { 9cca12e4,0 b832ebdd,0 } joined { } left { } partitioned { } ) 2019-01-07T06:37:36.393334Z 0 [Note] WSREP: Save the discovered primary-component to disk 2019-01-07T06:37:36.399450Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 2 2019-01-07T06:37:36.400149Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: b8812d8e-1246-11e9-91a9-ebbe7a4cd438 2019-01-07T06:37:36.403344Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: b8812d8e-1246-11e9-91a9-ebbe7a4cd438 2019-01-07T06:37:36.403590Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: b8812d8e-1246-11e9-91a9-ebbe7a4cd438 from 0 (pld1fmk0113) 2019-01-07T06:37:36.893044Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: b8812d8e-1246-11e9-91a9-ebbe7a4cd438 from 1 (pld1fmk0114) 2019-01-07T06:37:36.893106Z 0 [Note] WSREP: Quorum results: version = 4, component = PRIMARY, conf_id = 35, members = 1/2 (primary/total), act_id = 0, last_appl. = 0, protocols = 0/9/3 (gcs/repl/appl), group UUID = 9ccc59d2-04c4-11e9-81f3-a650bf4e5391 2019-01-07T06:37:36.893131Z 0 [Note] WSREP: Flow-control interval: [141, 141] 2019-01-07T06:37:36.893140Z 0 [Note] WSREP: Trying to continue unpaused monitor 2019-01-07T06:37:36.893287Z 3 [Note] WSREP: REPL Protocols: 9 (4, 2) 2019-01-07T06:37:36.893326Z 3 [Note] WSREP: New cluster view: global state: 9ccc59d2-04c4-11e9-81f3-a650bf4e5391:0, view# 36: Primary, number of nodes: 2, my index: 0, protocol version 3 2019-01-07T06:37:36.893340Z 3 [Note] WSREP: Setting wsrep_ready to true 2019-01-07T06:37:36.893350Z 3 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 2) 2019-01-07T06:37:36.893362Z 3 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification. 2019-01-07T06:37:36.893376Z 3 [Note] WSREP: Assign initial position for certification: 0, protocol version: 4 2019-01-07T06:37:36.893414Z 0 [Note] WSREP: Service thread queue flushed. 2019-01-07T06:37:37.576865Z 0 [Warning] WSREP: Member 1.0 (pld1fmk0114) requested state transfer from '172.19.101.107', but it is impossible to select State Transfer donor: No route to host 2019-01-07T06:37:38.577505Z 0 [Note] WSREP: forgetting b832ebdd (tcp://172.19.101.108:4567) 2019-01-07T06:37:38.577635Z 0 [Note] WSREP: Node 9cca12e4 state primary 2019-01-07T06:37:38.577672Z 0 [Note] WSREP: Current view of cluster as seen by this node view (view_id(PRIM,9cca12e4,37) memb { 9cca12e4,0 } joined { } left { } partitioned { b832ebdd,0 } ) 2019-01-07T06:37:38.577690Z 0 [Note] WSREP: Save the discovered primary-component to disk 2019-01-07T06:37:38.581164Z 0 [Note] WSREP: forgetting b832ebdd (tcp://172.19.101.108:4567) 2019-01-07T06:37:38.581199Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 1 2019-01-07T06:37:38.581856Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: b9ce1539-1246-11e9-b294-1a2193c07801 2019-01-07T06:37:38.581885Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: b9ce1539-1246-11e9-b294-1a2193c07801 2019-01-07T06:37:38.581897Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: b9ce1539-1246-11e9-b294-1a2193c07801 from 0 (pld1fmk0113) 2019-01-07T06:37:38.581908Z 0 [Note] WSREP: Quorum results: version = 4, component = PRIMARY, conf_id = 36, members = 1/1 (primary/total), act_id = 0, last_appl. = 0, protocols = 0/9/3 (gcs/repl/appl), group UUID = 9ccc59d2-04c4-11e9-81f3-a650bf4e5391 2019-01-07T06:37:38.581918Z 0 [Note] WSREP: Flow-control interval: [100, 100] 2019-01-07T06:37:38.581925Z 0 [Note] WSREP: Trying to continue unpaused monitor 2019-01-07T06:37:38.582032Z 3 [Note] WSREP: REPL Protocols: 9 (4, 2) 2019-01-07T06:37:38.582059Z 3 [Note] WSREP: New cluster view: global state: 9ccc59d2-04c4-11e9-81f3-a650bf4e5391:0, view# 37: Primary, number of nodes: 1, my index: 0, protocol version 3 2019-01-07T06:37:38.582068Z 3 [Note] WSREP: Setting wsrep_ready to true 2019-01-07T06:37:38.582076Z 3 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 2 -> 1) 2019-01-07T06:37:38.582085Z 3 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification. 2019-01-07T06:37:38.582096Z 3 [Note] WSREP: Assign initial position for certification: 0, protocol version: 4 2019-01-07T06:37:38.582124Z 0 [Note] WSREP: Service thread queue flushed. 2019-01-07T06:37:39.393373Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') turning message relay requesting off 2019-01-07T06:37:42.393896Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') connection established to b832ebdd tcp://172.19.101.108:4567 2019-01-07T06:37:42.393958Z 0 [Warning] WSREP: discarding established (time wait) b832ebdd (tcp://172.19.101.108:4567) 2019-01-07T06:37:43.893969Z 0 [Note] WSREP: cleaning up b832ebdd (tcp://172.19.101.108:4567) 2019-01-07T06:38:31.157156Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') connection established to d923a205 tcp://172.19.101.108:4567 2019-01-07T06:38:31.157527Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') turning message relay requesting on, nonlive peers: 2019-01-07T06:38:31.656713Z 0 [Note] WSREP: declaring d923a205 at tcp://172.19.101.108:4567 stable 2019-01-07T06:38:31.657097Z 0 [Note] WSREP: Node 9cca12e4 state primary 2019-01-07T06:38:31.657435Z 0 [Note] WSREP: Current view of cluster as seen by this node view (view_id(PRIM,9cca12e4,38) memb { 9cca12e4,0 d923a205,0 } joined { } left { } partitioned { } ) 2019-01-07T06:38:31.657462Z 0 [Note] WSREP: Save the discovered primary-component to disk 2019-01-07T06:38:31.661066Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 2 2019-01-07T06:38:31.661762Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: d9716ddc-1246-11e9-9ac7-43254a50de72 2019-01-07T06:38:31.664826Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: d9716ddc-1246-11e9-9ac7-43254a50de72 2019-01-07T06:38:31.665033Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: d9716ddc-1246-11e9-9ac7-43254a50de72 from 0 (pld1fmk0113) 2019-01-07T06:38:32.157306Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: d9716ddc-1246-11e9-9ac7-43254a50de72 from 1 (pld1fmk0114) 2019-01-07T06:38:32.157371Z 0 [Note] WSREP: Quorum results: version = 4, component = PRIMARY, conf_id = 37, members = 1/2 (primary/total), act_id = 0, last_appl. = 0, protocols = 0/9/3 (gcs/repl/appl), group UUID = 9ccc59d2-04c4-11e9-81f3-a650bf4e5391 2019-01-07T06:38:32.157387Z 0 [Note] WSREP: Flow-control interval: [141, 141] 2019-01-07T06:38:32.157396Z 0 [Note] WSREP: Trying to continue unpaused monitor 2019-01-07T06:38:32.157488Z 3 [Note] WSREP: REPL Protocols: 9 (4, 2) 2019-01-07T06:38:32.157519Z 3 [Note] WSREP: New cluster view: global state: 9ccc59d2-04c4-11e9-81f3-a650bf4e5391:0, view# 38: Primary, number of nodes: 2, my index: 0, protocol version 3 2019-01-07T06:38:32.157528Z 3 [Note] WSREP: Setting wsrep_ready to true 2019-01-07T06:38:32.157537Z 3 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 2) 2019-01-07T06:38:32.157528Z 3 [Note] WSREP: Setting wsrep_ready to true 2019-01-07T06:38:32.157537Z 3 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 1 -> 2) 2019-01-07T06:38:32.157546Z 3 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification. 2019-01-07T06:38:32.157557Z 3 [Note] WSREP: Assign initial position for certification: 0, protocol version: 4 2019-01-07T06:38:32.157595Z 0 [Note] WSREP: Service thread queue flushed. 2019-01-07T06:38:34.400423Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') turning message relay requesting off 2019-01-07T06:39:59.269712Z 0 [Warning] WSREP: Member 1.0 (pld1fmk0114) requested state transfer from '172.19.101.107', but it is impossible to select State Transfer donor: No route to host 2019-01-07T06:40:00.270193Z 0 [Note] WSREP: forgetting d923a205 (tcp://172.19.101.108:4567) 2019-01-07T06:40:00.270291Z 0 [Note] WSREP: Node 9cca12e4 state primary 2019-01-07T06:40:00.270325Z 0 [Note] WSREP: Current view of cluster as seen by this node view (view_id(PRIM,9cca12e4,39) memb { 9cca12e4,0 } joined { } left { } partitioned { d923a205,0 } ) 2019-01-07T06:40:00.270343Z 0 [Note] WSREP: Save the discovered primary-component to disk 2019-01-07T06:40:00.274052Z 0 [Note] WSREP: forgetting d923a205 (tcp://172.19.101.108:4567) 2019-01-07T06:40:00.274085Z 0 [Note] WSREP: New COMPONENT: primary = yes, bootstrap = no, my_idx = 0, memb_num = 1 2019-01-07T06:40:00.274743Z 0 [Note] WSREP: STATE_EXCHANGE: sent state UUID: 0e42b398-1247-11e9-aea6-339cafc13651 2019-01-07T06:40:00.274773Z 0 [Note] WSREP: STATE EXCHANGE: sent state msg: 0e42b398-1247-11e9-aea6-339cafc13651 2019-01-07T06:40:00.274784Z 0 [Note] WSREP: STATE EXCHANGE: got state msg: 0e42b398-1247-11e9-aea6-339cafc13651 from 0 (pld1fmk0113) 2019-01-07T06:40:00.274795Z 0 [Note] WSREP: Quorum results: version = 4, component = PRIMARY, conf_id = 38, members = 1/1 (primary/total), act_id = 0, last_appl. = 0, protocols = 0/9/3 (gcs/repl/appl), group UUID = 9ccc59d2-04c4-11e9-81f3-a650bf4e5391 2019-01-07T06:40:00.274806Z 0 [Note] WSREP: Flow-control interval: [100, 100] 2019-01-07T06:40:00.274812Z 0 [Note] WSREP: Trying to continue unpaused monitor 2019-01-07T06:40:00.274878Z 3 [Note] WSREP: REPL Protocols: 9 (4, 2) 2019-01-07T06:40:00.274899Z 3 [Note] WSREP: New cluster view: global state: 9ccc59d2-04c4-11e9-81f3-a650bf4e5391:0, view# 39: Primary, number of nodes: 1, my index: 0, protocol version 3 2019-01-07T06:40:00.274906Z 3 [Note] WSREP: Setting wsrep_ready to true 2019-01-07T06:40:00.274915Z 3 [Note] WSREP: Auto Increment Offset/Increment re-align with cluster membership change (Offset: 1 -> 1) (Increment: 2 -> 1) 2019-01-07T06:40:00.274923Z 3 [Note] WSREP: wsrep_notify_cmd is not defined, skipping notification. 2019-01-07T06:40:00.274947Z 3 [Note] WSREP: Assign initial position for certification: 0, protocol version: 4 2019-01-07T06:40:00.274978Z 0 [Note] WSREP: Service thread queue flushed. 2019-01-07T06:40:04.169757Z 0 [Note] WSREP: (9cca12e4, 'tcp://0.0.0.0:4567') connection established to d923a205 tcp://172.19.101.108:4567 2019-01-07T06:40:04.169812Z 0 [Warning] WSREP: discarding established (time wait) d923a205 (tcp://172.19.101.108:4567) 2019-01-07T06:40:05.410318Z 0 [Note] WSREP: cleaning up d923a205 (tcp://172.19.101.108:4567) ================================================================================================================= [root@pld1fmk0113 ~]# cat /etc/my.cnf [mysqld] server-id=1 datadir=/mysql/data socket=/mysql/mysql.sock log-error=/mysql/log/mysqld.log pid-file=/mysql/mysqld.pid log-bin=pld1fmk0113-bin log-bin-index=pld1fmk0113-bin.index max_binlog_size=500M log_slave_updates expire_logs_days=7 # Disabling symbolic-links is recommended to prevent assorted security risks symbolic-links=0 [mysqld_safe] pid-file = /mysql/mysqld.pid socket = /mysql/mysql.sock nice = 0 [mysqld] # Path to Galera library wsrep_provider=/usr/lib64/galera3/libgalera_smm.so table_open_cache=2000 explicit_defaults_for_timestamp = 1 #max_user_connections = 1000; # Cluster connection URL contains IPs of nodes #If no IP is found, this implies that a new cluster needs to be created, #in order to do that you need to bootstrap this node wsrep_cluster_address=gcomm://172.19.101.107,172.19.101.108 # In order for Galera to work correctly binlog format should be ROW binlog_format=ROW # MyISAM storage engine has only experimental support default_storage_engine=InnoDB # Slave thread to use wsrep_slave_threads= 8 wsrep_log_conflicts # This changes how InnoDB autoincrement locks are managed and is a requirement for Galera innodb_autoinc_lock_mode=2 # Node IP address wsrep_node_address=172.19.101.107 wsrep_sst_receive_address=172.19.101.107 # Cluster name wsrep_cluster_name=fmk113114 #If wsrep_node_name is not specified, then system hostname will be used wsrep_node_name=pld1fmk0113 #pxc_strict_mode allowed values: DISABLED,PERMISSIVE,ENFORCING,MASTER pxc_strict_mode=PERMISSIVE # SST method wsrep_sst_method=xtrabackup-v2 #Authentication for SST method wsrep_sst_auth="sstuser:sstpass"