‎Administer failover script

Configuration

failover.py

Configuration file path: /opt/workfusion/failover/conf/SERVICE.ini

See failover.py configuration example

[default]
; Script checks every x seconds that all child processes are up and running
; 'process_watchdog_period' defines timeout between checks
; Usually, you don't need to change this parameter
process_watchdog_period = 3


[environment]
; Script stores metadata in ZooKeeper
; 'environment_id' and 'environment_region' properties are used to generate metadata znode path
; Usually, you don't need to change this parameters  
environment_id = <environment_id>
environment_region = <environment_region>


[service]
; Script uses service name for execution of service-specific actions and logging
; You don't need to change this parameter
service_id = <service_id>


[application_process_rest_api]
; Script exposes HTTP endpoints for integration and maintenance
; You could define ip and port on which this endpoint is available 
flask_host = <flask_bind_ip>
flask_port = <flask_bind_port>


[application_process_health_check]
; Script executes service HTTP health checks
; Requests executed to the 'service_check_url' endpoint, script expects to receive the HTTP 200 status code response in 'execution_timeout' seconds
; After each check, there is a pause for 'execution_period' seconds
; Usually, you don't need to change this parameters
execution_period = 10
execution_timeout = 10
service_check_url = <service_check_url>

[application_process_leader_election]
; Script elects cluster leader periodically
; Script tries to execute leader election actions for 'execution_timeout' seconds. After each try, there is a pause for 'execution_period' seconds
; If a node is a leader and has passed health and status checks, it prolongs the leader lock for 'leader_znode_lock_time_seconds' seconds
; If a node is a follower and it takes a leader lock, the node is promoted. For next 'service_start_counter' cycles, health and status checks are skipped. Script will wait for the service to start
; If a node is a leader and health or status checks failed, 'cluster_unstable_counter' is decremented each failed cycle
; When cluster_unstable_counter becomes equal 0, the node is demoted to a follower and does not execute leader election action for 'service_start_counter' cycles
; When health and status checks passed 'cluster_unstable_counter' resets to the initial value
; Usually, you don't need to change this parameters
execution_period = 10
execution_timeout = 10
service_start_counter = 10
cluster_unstable_counter = 15
leader_znode_lock_time_seconds = 50


[application_process_manager]
; Script checks the service status periodically and starts or stops it, depending on the node role in the cluster 
; Script tries to execute actions for 'execution_timeout' seconds. After each try, there is a pause for 'execution_period' seconds
; In addition, for Nexus, you can enable or disable the scheduling of the artifact index rebuild job; If enabled, it scheduled after each node promotion from a follower to the leader
; Usually, you don't need to change these parameters
execution_period = 10
execution_timeout = 10
nexus_reindex_all_repo = disabled|enabled


[application_process_status_check]
; Script executes watchdog.py HTTP health checks
; Requests executed to 'watchdog_endpoint' endpoint, script expects to receive the HTTP 200 status code response in 'execution_timeout' seconds
; After each check, there is a pause for 'execution_period' seconds
; Usually, you don't need to change this parameters
watchdog_endpoint = <watchdog_endpoint>
execution_period = 10
execution_timeout = 10


[supervisord]
; Script uses supervisord for process starts and stops
; You don't need to change this parameters
supervisorctl_update_cmd = <supervisorctl_update_cmd>
supervisord_endpoint = <supervisord_endpoint>
supervisord_app_configuration_path = <supervisord_app_configuration_path>
supervisord_app_configuration_extension = ini


[nexus]
; Script could schedule an index update job in Nexus
; You could defile Nexus credentials and endpoint in the raw format (nexus_user = nexus) or provide a ZooKeeper znode path (nexus_user = zk:///failover/test/dc1/failover/configuration/nexus/nexus_user)
; Usually, you don't need to change this parameters
nexus_endpoint = <nexus_endpoint>
nexus_user = <nexus_user>
nexus_password = <nexus_password>


[zookeeper]
; Script stores metadata in ZooKeeper
; You could configure the script to use another Zookeeper cluster or instance
zookeeper_connection_string = <zookeeper_endpoint>

watchdog.py

Configuration file path: /opt/workfusion/failover/conf/watchdog-conf.ini

See watchdog.py configuration example

[default]
; Script checks every x seconds that all child processes are up and running
; 'process_watchdog_period' defines the timeout between checks
; Usually, you don't need to change this parameter
process_watchdog_period = 3


[environment]
; Script stores metadata in ZooKeeper
; The 'environment_id' and 'environment_region' properties are used to generate metadata znode path
; Usually, you don't need to change this parameters  
environment_id = <environment_id>
environment_region = <environment_region>


[supervisord]
; Script uses supervisord for process starts and stops
; You don't need to change this parameters
supervisorctl_update_cmd = <supervisorctl_update_cmd>
supervisord_endpoint = <supervisord_endpoint>
supervisord_app_configuration_path = <supervisord_app_configuration_path>
supervisord_app_configuration_extension = ini


[application_process_dependencies]
; Script checks node dependencies periodically; the check takes up to 'execution_timeout' seconds
; After each check, there is a pause for 'execution_period' seconds
; Usually, you don't need to change these parameters
execution_period = 20
execution_timeout = 10

[application_process_watchdog]
; Script checks failover.py periodically; the check takes up to 'execution_timeout' seconds
; If the failover.py check failed and the corresponding service is running, watchdog.py stops the service 
; After each check, there is a pause for 'execution_period' seconds
; Usually, you don't need to change these parameters
execution_period = 20
execution_timeout = 10

[application_process_rest_api]
; Script exposes HTTP endpoints for integration and maintenance
; You could define ip and port on which this endpoint is available 
flask_host = <flask_bind_ip>
flask_port = <flask_bind_port>

[watch_dependencies]
; List of dependencies to be monitored by watchdog.py
; Usually, you don't need to change this parameters
mssql = enabled|disabled
vault = enabled|disabled
nginx = enabled|disabled
haproxy = enabled|disabled
zookeeper = enabled|disabled

[mssql]
; MSSQL monitoring properties
; You don't need to change this parameters
check_tcp = disabled
check_tcp_host = <mssql_hostname>
check_tcp_port = <mssql_port>

[vault]
; Vault monitoring properties
; You don't need to change this parameters
check_tcp = disabled
check_tcp_host = <vault_ip>
check_tcp_port = <vault_port>
check_srv = disabled
check_srv_http_url = <vault_healthcheck_url>

[nginx]
; Nginx monitoring properties
; You don't need to change this parameters
check_tcp = disabled
check_tcp_host = <nginx_ip>
check_tcp_port = <nginx_port>
check_srv = disabled
check_srv_http_url = <nginx_healthcheck_url>

[haproxy]
; HAProxy monitoring properties
; You don't need to change this parameters
check_tcp = disabled
check_tcp_host = <haproxy_ip>
check_tcp_port = <haproxy_port>
check_srv = disabled
check_srv_http_url = <haproxy_healthcheck_url>

[zookeeper]
; ZooKeeper monitoring properties
; You don't need to change this parameters
check_tcp = disabled
check_tcp_host = <zookeeper_leader_host>
check_tcp_port = <zookeeper_leader_client_port>
check_srv = disabled
zookeeper_connection_string = <zookeeper_connection_string>

[watch_services]
; List of services to be monitored by watchdog.py
; Usually, you don't need to change this parameters
nexus = enabled|disabled
task-dispatcher-service = enabled|disabled
automl-gateway-service = enabled|disabled
automl-model-service = enabled|disabled
automl-model-management-service = enabled|disabled
worker-management-service = enabled|disabled
workfusion = enabled|disabled
workspace = enabled|disabled
bot-manager = enabled|disabled
sqc = enabled|disabled

[nexus]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[task-dispatcher-service]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[automl-gateway-service]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[automl-model-service]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[automl-model-management-service]
; Endpoint to monitor failover.py availability 
; Usually to don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[worker-management-service]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[workfusion]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[workspace]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[bot-manager]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

[sqc]
; Endpoint to monitor failover.py availability 
; Usually, you don't need to change this parameter
monitoring_endpoint = <monitoring_endpoint>

Maintenance

To perform maintenance of the service managed by failover.py, follow the steps below:

Enable the maintenance mode.

Get the failover.py and watchdog.py endpoints in the [application_process_rest_api] block of the /opt/workfusion/failover/conf/SERVICE.ini and /opt/workfusion/failover/conf/watchdog-conf.ini configuration files.

Enable the maintenance mode for watchdog.py.

curl -lv http://127.0.0.1:10500/api/v1/maintenance/enable

Example output:

* About to connect() to 127.0.0.1 port 10500 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10500 (#0)
> GET /api/v1/maintenance/enable HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10500
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 19
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:29:45 GMT
< 
* Closing connection 0
maintenance-enabled

Enable the maintenance mode for failover.py.

curl -lv http://127.0.0.1:<failover.py port>/api/v1/maintenance/enable

Example output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10501/api/v1/maintenance/enable* About to connect() to 127.0.0.1 port 10501 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10501 (#0)
> GET /api/v1/maintenance/enable HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10501
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 19
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:30:52 GMT
< 
* Closing connection 0
maintenance-enabled

Perform maintenance operations.

Disable the maintenance mode:

Disable the maintenance mode for failover.py.

curl -lv http://127.0.0.1:<failover.py port>/api/v1/maintenance/enable

Example output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10501/api/v1/maintenance/disable
* About to connect() to 127.0.0.1 port 10501 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10501 (#0)
> GET /api/v1/maintenance/disable HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10501
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 20
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:31:48 GMT
< 
* Closing connection 0
maintenance-disabled

Disable the maintenance mode for watchdog.py:

curl -lv http://127.0.0.1:10500/api/v1/maintenance/disable

Example output:

curl -lv http://127.0.0.1:10500/api/v1/maintenance/disable
* About to connect() to 127.0.0.1 port 10500 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10500 (#0)
> GET /api/v1/maintenance/disable HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10500
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 20
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:30:08 GMT
< 
* Closing connection 0
maintenance-disabled

Troubleshooting

Logging

The log file for failover.py is /opt/workfusion/supervisord/log/failover-SERVICE.log.
The log file for watchdog.py is /opt/workfusion/supervisord/log/watchdog.log.

Dependency checks

Supervisord

Check that Supervisord API is available on port 9900.

netstat -lnpt | grep 9900

Expected output:

tcp 0 0 127.0.0.1:9900 0.0.0.0:* LISTEN 2812/python

ZooKeeper

Check ZooKeeper availability.

echo mntr | nc <zookeeper_server_ip>  <zookeeper_server_port>

Expected output:

[root@ip-10-100-22-158 conf]# echo mntr | nc 10.100.22.158 2181
zk_version 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT
zk_avg_latency 0
zk_max_latency 281
zk_min_latency 0
zk_packets_received 737229
zk_packets_sent 738126
zk_num_alive_connections 159
zk_outstanding_requests 0
zk_server_state standalone
zk_znode_count 1436
zk_watch_count 2127
zk_ephemerals_count 6
zk_approximate_data_size 980411
zk_open_file_descriptor_count 185
zk_max_file_descriptor_count 200000
zk_fsync_threshold_exceed_count 0

Endpoint checks

To check the failover.py endpoints, follow the steps below:

Check the monitoring endpoint.

curl -lv http://127.0.0.1:<failover.py port>/api/v1/monitoring

Expected output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10501/api/v1/monitoring
* About to connect() to 127.0.0.1 port 10501 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10501 (#0)
> GET /api/v1/monitoring HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10501
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 2
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:21:15 GMT
< 
* Closing connection 0
OK

Check the service health.

curl -lv http://127.0.0.1:<failover.py port>/api/v1/health

Expected output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10501/api/v1/health
* About to connect() to 127.0.0.1 port 10501 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10501 (#0)
> GET /api/v1/health HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10501
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 51
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:21:57 GMT
< 
* Closing connection 0
[2019-08-29 12:21:43][OK](Leader) HealthCheckPassed

Check the node status.

curl -lv http://127.0.0.1:<failover.py port>/api/v1/status

Expected output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10501/api/v1/status
* About to connect() to 127.0.0.1 port 10501 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10501 (#0)
> GET /api/v1/status HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10501
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 43
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:23:47 GMT
< 
* Closing connection 0
[2019-08-29 12:23:44][OK] StatusCheckPassed

To check the watchdog.py monitoring endpoint, run the following command:

curl -lv http://127.0.0.1:10500/api/v1/monitoring

Expected output:

[root@ip-10-100-22-158 conf]# curl -lv http://127.0.0.1:10500/api/v1/monitoring
* About to connect() to 127.0.0.1 port 10500 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 10500 (#0)
> GET /api/v1/monitoring HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:10500
> Accept: */*
> 
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
< Content-Type: text/html; charset=utf-8
< Content-Length: 26
< Server: Werkzeug/0.15.5 Python/2.7.5
< Date: Thu, 29 Aug 2019 12:18:46 GMT
< 
* Closing connection 0
2019-08-29 12:18:40.755086[root@ip-10-100-22-158 conf]

WorkFusion Documentation

10.1.2

Performance dashboards

Control Tower

Analytics

Business Process

Database

ELK

Installation

Infrastructure

AutoML

Logs

RPA

OCR

BEP

S3 File Storage

Studio

Configuration

failover.py

watchdog.py

Maintenance

Troubleshooting

Logging

Dependency checks

Supervisord

ZooKeeper

Endpoint checks