From df17a06865947a24d24fccf92543b5d185c0eac0 Mon Sep 17 00:00:00 2001 From: Ross Paterson Date: Thu, 25 Aug 2016 14:07:19 +0100 Subject: [PATCH 1/3] revised initial node health tool commit --- clearwater-infrastructure/usr/bin/health-tool | 3 + .../usr/bin/health-tool_source/checkers.py | 295 +++++++ .../usr/bin/health-tool_source/fetchers.py | 164 ++++ .../usr/bin/health-tool_source/run.py | 278 ++++++ .../usr/bin/health-tool_source/snmpy.py | 30 + .../usr/bin/health-tool_source/viewers.py | 803 ++++++++++++++++++ 6 files changed, 1573 insertions(+) create mode 100644 clearwater-infrastructure/usr/bin/health-tool create mode 100644 clearwater-infrastructure/usr/bin/health-tool_source/checkers.py create mode 100644 clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py create mode 100644 clearwater-infrastructure/usr/bin/health-tool_source/run.py create mode 100644 clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py create mode 100644 clearwater-infrastructure/usr/bin/health-tool_source/viewers.py diff --git a/clearwater-infrastructure/usr/bin/health-tool b/clearwater-infrastructure/usr/bin/health-tool new file mode 100644 index 00000000..4cc91571 --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool @@ -0,0 +1,3 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; +$DIR/run.py \ No newline at end of file diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/checkers.py b/clearwater-infrastructure/usr/bin/health-tool_source/checkers.py new file mode 100644 index 00000000..fd508bd2 --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool_source/checkers.py @@ -0,0 +1,295 @@ +#!/usr/bin/python3 +'''The set of checker classes for HealthTool''' +import logging +import subprocess +import snmpy +import time +import re +import collections +from fetchers import KillableObject + + +snmp = snmpy.SubProcessSNMP('clearwater', 'localhost') +logger = logging.getLogger(__name__) + +INFO_DICT = {'CPU USE': 'YOUR CPU USE APPEARS TO BE TOO HIGH', + 'DISK USE': 'YOUR DISK USE APPEARS TO BE TOO HIGH', + 'ETCD CLUSTER': 'ETCD NOT YET FULLY CLUSTERED', + 'MEMCACHED CLUSTER': 'MEMCACHED NOT YET FULLY CLUSTERED', + 'CHRONOS CLUSTER': 'CHRONOS NOT YET FULLY CLUSTERED', + 'CASSANDRA CLUSTER': 'CASSANDRA NOT YET FULLY CLUSTERED', + 'NODE': 'ONE OR MORE NODE PROCESSES NOT YET HEALTHY' + } + + +class CheckerController(KillableObject): + def __init__(self, first_checker, health_group): + self.alive = True + self.first_checker = first_checker + self.health_group = health_group + + def run(self): + self.health_group.display_loading_message() + self.first_checker.run() + self.health_group.remove_loading_message() + # Variable necessary for testing + loop_alive = True + while loop_alive: + if self.alive: + self.health_group.update() + self.health_group.wipe_info() + time.sleep(20) + self.first_checker.run() + + +class Checker(object): + '''This is the abstract superclass for all of the checker objects for the tool''' + def __init__(self, node_name, node_version, health_group, next_object=None): + ''' + Args: + node_name: The name of the node in question + node_version: The version of the node in question + next_object: The next checker in the chain of responsibility + health_group: The health group that the checkers are to interact with + ''' + logger.info('Creating new checker') + self.next_object = next_object + self.health_group = health_group + self.node_name = node_name + self.node_version = node_version + + def run(self): + ''' + performs the health check, quits the chain of responsibility if the check + returns false, else it will execute the run method of the next checker + ''' + continue_chain = True + logger.info('Running checker') + continue_chain = self.health_check() + if self.next_object and continue_chain: + return self.next_object.run() + else: + return True + + def health_check(self): + logger.exception('Checker.health_check was called for abstract class') + raise NotImplementedError('An abstract method was called.') + + +class UsageChecker(Checker): + ''' A class for checking CPU and Disk use on the node, these checks are not considered massively + important and as such return warnings for the user rather than errors. + ''' + def __init__(self, node_name, node_version, health_group, next_object=None): + ''' + Args: + node_name: The name of the node in question + node_version: The version of the node in question + next_object: The next checker in the chain of responsibility + health_group: The health group that the checkers are to interact with + ''' + super().__init__(node_name, node_version, health_group, next_object) + self.MAX_DISK = 90 + self.MAX_CPU = 60 + self.system_oids = {'idle': '.1.3.6.1.4.1.2021.11.53.0', + 'raw_user': '.1.3.6.1.4.1.2021.11.50.0', + 'nice': '.1.3.6.1.4.1.2021.11.51.0', + 'system': '.1.3.6.1.4.1.2021.11.52.0', + 'interrupts': '.1.3.6.1.4.1.2021.11.56.0', + } + + def health_check(self): + logger.info('Checking disk usage') + self.check_disk_use() + logger.info('Checking CPU usage') + self.check_cpu_use() + return True + + def check_disk_use(self): + ''' + Checks the disk use by running landscape-sysinfo and then taking the percentage and + comparing to the maximum set in the health_check method + ''' + system_info = subprocess.check_output( + 'landscape-sysinfo', shell=True).decode('UTF-8') + match = re.search(r'Usage of /: *?(?P.*?)% of', system_info) + disk_use = match.group('disk_use') + disk_use = float(disk_use) + logger.info('found disk use of %s', disk_use) + if disk_use > self.MAX_DISK: + self.health_group.add_warning_info('DISK USE', INFO_DICT['DISK USE']) + self.health_group.health_addition('DISK USE', 'WARNING') + else: + self.health_group.health_addition('DISK USE', 'GOOD') + + def check_cpu_use(self): + ''' + Checks the CPU use by using the SNMP statistics for each kind of CPU activity, + using them to find out how long is spent idle and then from there deciding percentage + use. + ''' + initial_ticks = self.get_total_ticks() + initial_idle = snmp.get(self.system_oids['idle']) + # sleeping for less than 7 seconds can cause there to have been no successful update and + # the program exits on a 'divides by 0 error', more than 7 causes the user to wait. + time.sleep(7) + final_ticks = self.get_total_ticks() + final_idle = snmp.get(self.system_oids['idle']) + total_ticks = final_ticks - initial_ticks + total_idle = final_idle - initial_idle + idle_percent = 100 * float(total_idle) / total_ticks + logger.debug( + 'Generated CPU idle percentages: %s', idle_percent) + cpu_use = 100 - idle_percent + logger.debug('CPU use is rated at: %s', cpu_use) + + if cpu_use > self.MAX_CPU: + self.health_group.add_warning_info('CPU USE', INFO_DICT['CPU USE']) + self.health_group.health_addition('CPU USE', 'WARNING') + else: + self.health_group.health_addition('CPU USE', 'GOOD') + return True + + def get_total_ticks(self): + raw_user = snmp.get(self.system_oids['raw_user']) + nice = snmp.get(self.system_oids['nice']) + system = snmp.get(self.system_oids['system']) + idle = snmp.get(self.system_oids['idle']) + interrupts = snmp.get(self.system_oids['interrupts']) + total_ticks = raw_user + nice + system + idle + interrupts + return total_ticks + + +class ClusterChecker(Checker): + ''' This is a class for the checker which assesses the clusters that the node is in and reports to the + viewer accordingly + ''' + + def health_check(self): + ''' The CLUSTER_ACTION_DICT is structured as below to ensure that the checker can understand Hierarchy + The first list of functions is the first 'Tier' etc.. So once something in a tier errors, + the elements in lower tiers will not be executed + ''' + SPROUT_TIER1 = [self.etcd_check] + SPROUT_TIER2 = [self.memcached_check, self.chronos_check] + RALF_TIER1 = [self.etcd_check] + RALF_TIER2 = [self.memcached_check, self.chronos_check] + HOMESTEAD_TIER1 = [self.etcd_check] + HOMESTEAD_TIER2 = [self.cassandra_check] + + CLUSTER_ACTION_DICT = { + 'sprout': [SPROUT_TIER1, SPROUT_TIER2], + 'ralf': [RALF_TIER1, RALF_TIER2], + 'homestead': [HOMESTEAD_TIER1, HOMESTEAD_TIER2] + } + + for tier in CLUSTER_ACTION_DICT[self.node_name]: + logger.debug('Entering new cluster tier') + tier_failed = False in [check() for check in tier] + if tier_failed: + logger.debug('A tier failed') + break + return True + + def etcd_check(self): + ''' Checks etcd health by running the cluster-health command and looking for the + string 'cluster is healthy'. If this string is not present, it will write an error + to the tools health interface. If the cluster is healthy, the tool will check if + any remote nodes are displaying poor health in the etcd cluster and display a + warning if they are. + ''' + etcd_health = subprocess.check_output( + 'clearwater-etcdctl cluster-health', shell=True).decode('UTF-8') + is_healthy = 'cluster is healthy' in etcd_health + if not is_healthy: + self.health_group.add_error_info('ETCD CLUSTER', INFO_DICT['ETCD CLUSTER']) + self.health_group.health_addition('ETCD CLUSTER', 'ERROR') + return is_healthy + else: + etcd_health = etcd_health.split('\n') + + for line in etcd_health: + match = re.search('member .* is (?P\w+)', line) + if match and match.group('health') != 'healthy': + logger.debug('found health at %s', match.group('health')) + self.health_group.add_warning_info('ETCD CLUSTER', INFO_DICT['ETCD CLUSTER']) + self.health_group.health_addition('ETCD CLUSTER', 'WARNING') + return False + + self.health_group.health_addition('ETCD CLUSTER', 'GOOD') + return True + + def memcached_check(self): + is_healthy = self.check_cluster_state('Memcached') + return is_healthy + + def chronos_check(self): + is_healthy = self.check_cluster_state('Chronos') + return is_healthy + + def cassandra_check(self): + is_healthy = self.check_cluster_state('Cassandra') + return is_healthy + + def check_cluster_state(self, cluster_type): + ''' Runs the check_cluster_state command and checks if the relevant cluster type for this node + is stable, writes the outcome to the viewer. + Args: + cluster_type: the string name of the cluster, e.g. 'Memcached' + ''' + cluster_state = subprocess.check_output('/usr/share/clearwater/clearwater-cluster-manager/scripts/check_cluster_state', shell=True).decode('UTF-8') + cluster_state = cluster_state.split('\n') + cluster_name = self.node_name.title() + ' ' + cluster_type.title() + + for line in cluster_state: + if cluster_name in line: + current_index = cluster_state.index(line) + # 2 is necessary due to the structure of the command output + is_healthy = 'The cluster is stable' in cluster_state[current_index + 2] + name_string = cluster_type.upper() + ' CLUSTER' + + if not is_healthy: + self.health_group.add_error_info(name_string, INFO_DICT[name_string]) + self.health_group.health_addition(name_string, 'ERROR') + else: + self.health_group.health_addition(name_string, 'GOOD') + return is_healthy + + +class NodeChecker(Checker): + ''' This is a class for a checker to assess the health of the node and report the result to the viewer. + ''' + + def __init__(self, node_name, node_version, health_group, next_object=None): + super().__init__(node_name, node_version, health_group, next_object) + # Defaults to True to ensure that 'waiting' is viewed as healthy UNTIL it has had a + # bad health signal beforehand. + self.monit_state_dictionary = collections.defaultdict(lambda: True) + + def health_check(self): + self.healthy_states = ['Running', 'Status ok', 'Accessible', 'Waiting'] + self.keywords = ['Process', 'Program', 'System', 'File', 'Fifo', 'Filesystem', 'Directory', 'Remote'] + is_healthy = self.monit_check() + return is_healthy + + def monit_check(self): + is_healthy = True + monit_info = subprocess.check_output('monit summary', shell=True).decode('UTF-8') + monit_lines = monit_info.split('\n') + for line in monit_lines: + for keyword in self.keywords: + if line.startswith(keyword): + match = re.search(keyword + " '(?P.*)' *(?P.*)", line) + if match.group('state') not in self.healthy_states: + self.monit_state_dictionary[match.group('name')] = False + elif match.group('state') != 'Waiting': + self.monit_state_dictionary[match.group('name')] = True + if False in self.monit_state_dictionary.values(): + is_healthy = False + + if not is_healthy: + self.health_group.add_error_info('NODE', INFO_DICT['NODE']) + self.health_group.health_addition('NODE', 'ERROR') + else: + self.health_group.health_addition('NODE', 'GOOD') + return is_healthy diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py b/clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py new file mode 100644 index 00000000..9da5d7dc --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py @@ -0,0 +1,164 @@ +''' The fetcher classes for the table of stats in the health tool''' +import logging +import snmpy +import time +import re +import os + + +logger = logging.getLogger(__name__) + + +class KillableObject(object): + ''' The abstract class for objects which can be killed or resurrected''' + def __init__(self): + raise NotImplementedError('KillableObject.__init__ has been called') + + def kill(self): + self.alive = False + + def resurrect(self): + self.alive = True + + +class Fetcher(KillableObject): + ''' The abstract class for Fetchers + ''' + def __init__(self, table_window, timer): + ''' + Args: + table_window: The TableWindow object from the viewer class which will be displaying + the results of the fetcher + timer: The current value for the refresh timer (this decides how often stats + are reported and over what period) + ''' + logger.info('Creating new fetcher') + self.table_window = table_window + self.timer = timer + self.alive = True + + def run(self): + self.display_loading_message() + stats = self.assemble_data() + self.remove_loading_message() + while True: + if self.alive: + self.display_data(stats) + stats = self.assemble_data() + + def assemble_data(self): + logger.exception('Fetcher.assemble_data was called for abstract class') + raise NotImplementedError('An abstract method was called.') + + def fetch_data(self): + logger.exception('Fetcher.fetch_data was called for abstract class') + raise NotImplementedError('An abstract method was called.') + + def display_data(self, data): + self.table_window.write_to_table(data) + + def update_timer(self, timer): + self.timer = timer + + def display_loading_message(self): + self.table_window.display_loading_message() + + def remove_loading_message(self): + self.table_window.remove_loading_message() + + +class StatFetcher(Fetcher): + + def __init__(self, table_window, timer, oid_dictionary): + ''' Args: + table_window: The TableWindow object from the viewer class which will be displaying + the results of the fetcher + timer: The current value for the refresh timer (this decides how often stats + are reported and over what period) + oid_dictionary: An OrderedDict() where each key contains either a single + OID or a list in the form [initial_segment, list_of_endings] where the list is to + be summed before display and displayed as a whole. + e.g. if you wish to display the sum of 1.3.1, 1.3.4, 1.3.5, 1.3.0.1 you simply put + ['.1.3.', [1, 4, '5', '0.1']], the endings are allowed to be integers. + ''' + super().__init__(table_window, timer) + self.snmp = snmpy.SubProcessSNMP('clearwater', 'localhost') + self.oid_dictionary = oid_dictionary + + def assemble_data(self): + ''' Uses the timer value to fetch 5 second data every 5 seconds until the timer is fulfilled and then + returns the total count of each variable + ''' + timer = self.timer + time_remaining = timer + stats = self.fetch_data() + time.sleep(5) + time_remaining -= 5 + while time_remaining > 0: + # data should be a list of lists in form [[name, value], [name, value], ...] + data = self.fetch_data() + stats = [[stats[0], x + y] for x, y in zip(stats[1], data[1])] + time.sleep(5) + time_remaining -= 5 + return stats + + def fetch_data(self): + ''' obtains 5 second period data using the snmp.snmpy classes + ''' + stats = [] + for name in self.oid_dictionary: + value = 0 + for oid in self.oid_dictionary[name]: + logger.debug('fetching oid: %s for name: %s', oid, name) + value += self.snmp.get(oid) + stats.append([name, value]) + return stats + + +class RalfFetcher(Fetcher): + ''' A checker to assess the load of a ralf node and output it's 2 statistics using the access logs. + ''' + def __init__(self, table_window, timer): + super().__init__(table_window, timer) + self.access_log = open('/var/log/ralf/access_current.txt', 'r') + self.access_log.seek(0, 2) + self.previous_file_name = self.get_canonical_path('/var/log/ralf/access_current.txt') + + def get_canonical_path(self, path): + return os.path.abspath(os.readlink(path)) + + def fetch_data(self, timer): + ''' fetches ralf data for the past timer value of seconds, using the access logs. + ''' + stats = [] + time.sleep(timer) + current_file_name = self.get_canonical_path('/var/log/ralf/access_current.txt') + lines = self.access_log.readlines() + + success_count = self.get_count_for(' 2.*? POST /call-id/', lines) + total_count = self.get_count_for(' ... POST /call-id/', lines) + + if current_file_name != self.previous_file_name: + self.access_log.close() + self.access_log = open('/var/log/ralf/access_current.txt', 'r') + lines = self.access_log.readlines() + + success_count += self.get_count_for(' 2.*? POST /call-id/', lines) + total_count += self.get_count_for(' ... POST /call-id/', lines) + + self.previous_file_name = current_file_name + stats.append(['Successful billing events', success_count]) + stats.append(['Total billing events', total_count]) + return stats + + def assemble_data(self): + ''' Wraps over the top of the fetch_data method to keep consistency with the superclass + ''' + return self.fetch_data(self.timer) + + def get_count_for(self, express, line_list): + ''' searches a list for an expression and returns the number of elements containing the expression. + ''' + expression = re.compile(express) + good_stuff = list(filter(expression.search, line_list)) + return len(good_stuff) diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/run.py b/clearwater-infrastructure/usr/bin/health-tool_source/run.py new file mode 100644 index 00000000..94a9edbc --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool_source/run.py @@ -0,0 +1,278 @@ +#!/usr/bin/python3 +''' The run file for the node health checker.''' +import collections +import logging +import os +import re +import subprocess +import sys +import threading +import checkers +import fetchers +import viewers + +logger = logging.getLogger(__name__) +# GLOBAL CONSTANT VARIABLES +TIMER_DEFAULT = 5 +NODE_LIST = ['sprout', 'homestead', 'ralf'] +SPROUT_OID_DICT = collections.OrderedDict() +HOMESTEAD_OID_DICT = collections.OrderedDict() +RALF_OID_DICT = collections.OrderedDict() + +# OID dictionarys should have single elements under each key or a list in the form: +# [initial_segment, list_of_endings] +# the initialisation below is necessary in order to have the required +# consistent ordering +SPROUT_OID_DICT['Incoming SIP requests'] = ['.1.2.826.0.1.1578918.9.3.6.1.2.1'] +SPROUT_OID_DICT[ + 'Registration Overload Rejections'] = ['.1.2.826.0.1.1578918.9.3.7.1.2.1'] +SPROUT_OID_DICT[ + 'Initial Registration Successes'] = ['.1.2.826.0.1.1578918.9.3.9.1.3.1'] +SPROUT_OID_DICT[ + 'Re-Registration Successes'] = ['.1.2.826.0.1.1578918.9.3.10.1.3.1'] +SPROUT_OID_DICT['ICSCF incoming SIP Successes'] = [ + '.1.2.826.0.1.1578918.9.3.18.1.4.1.0', '.1.2.826.0.1.1578918.9.3.18.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.2', '.1.2.826.0.1.1578918.9.3.18.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.4', '.1.2.826.0.1.1578918.9.3.18.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.6', '.1.2.826.0.1.1578918.9.3.18.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.8', '.1.2.826.0.1.1578918.9.3.18.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.10', '.1.2.826.0.1.1578918.9.3.18.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.12', '.1.2.826.0.1.1578918.9.3.18.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.18.1.4.1.14'] +SPROUT_OID_DICT['ICSCF outgoing SIP Successes'] = [ + '.1.2.826.0.1.1578918.9.3.19.1.4.1.0', '.1.2.826.0.1.1578918.9.3.19.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.2', '.1.2.826.0.1.1578918.9.3.19.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.4', '.1.2.826.0.1.1578918.9.3.19.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.6', '.1.2.826.0.1.1578918.9.3.19.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.8', '.1.2.826.0.1.1578918.9.3.19.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.10', '.1.2.826.0.1.1578918.9.3.19.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.12', '.1.2.826.0.1.1578918.9.3.19.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.19.1.4.1.14'] +SPROUT_OID_DICT['SCSCF incoming INVITE Successes'] = ['.1.2.826.0.1.1578918.9.3.20.1.4.1.0'] +SPROUT_OID_DICT['SCSCF incoming NON-INVITE Successes'] = [ + '.1.2.826.0.1.1578918.9.3.20.1.4.1.0', '.1.2.826.0.1.1578918.9.3.20.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.2', '.1.2.826.0.1.1578918.9.3.20.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.4', '.1.2.826.0.1.1578918.9.3.20.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.6', '.1.2.826.0.1.1578918.9.3.20.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.8', '.1.2.826.0.1.1578918.9.3.20.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.10', '.1.2.826.0.1.1578918.9.3.20.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.12', '.1.2.826.0.1.1578918.9.3.20.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.20.1.4.1.14'] +SPROUT_OID_DICT['SCSCF outgoing SIP Successes'] = [ + '.1.2.826.0.1.1578918.9.3.21.1.4.1.0', '.1.2.826.0.1.1578918.9.3.21.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.2', '.1.2.826.0.1.1578918.9.3.21.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.4', '.1.2.826.0.1.1578918.9.3.21.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.6', '.1.2.826.0.1.1578918.9.3.21.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.8', '.1.2.826.0.1.1578918.9.3.21.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.10', '.1.2.826.0.1.1578918.9.3.21.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.12', '.1.2.826.0.1.1578918.9.3.21.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.21.1.4.1.14'] +SPROUT_OID_DICT['BGCF incoming SIP Successes'] = [ + '.1.2.826.0.1.1578918.9.3.22.1.4.1.0', '.1.2.826.0.1.1578918.9.3.22.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.2', '.1.2.826.0.1.1578918.9.3.22.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.4', '.1.2.826.0.1.1578918.9.3.22.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.6', '.1.2.826.0.1.1578918.9.3.22.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.8', '.1.2.826.0.1.1578918.9.3.22.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.10', '.1.2.826.0.1.1578918.9.3.22.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.12', '.1.2.826.0.1.1578918.9.3.22.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.22.1.4.1.14'] +SPROUT_OID_DICT['BGCF outgoing SIP Successes'] = [ + '.1.2.826.0.1.1578918.9.3.23.1.4.1.0', '.1.2.826.0.1.1578918.9.3.23.1.4.1.1', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.2', '.1.2.826.0.1.1578918.9.3.23.1.4.1.3', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.4', '.1.2.826.0.1.1578918.9.3.23.1.4.1.5', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.6', '.1.2.826.0.1.1578918.9.3.23.1.4.1.7', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.8', '.1.2.826.0.1.1578918.9.3.23.1.4.1.9', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.10', '.1.2.826.0.1.1578918.9.3.23.1.4.1.11', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.12', '.1.2.826.0.1.1578918.9.3.23.1.4.1.13', + '.1.2.826.0.1.1578918.9.3.23.1.4.1.14'] + +HOMESTEAD_OID_DICT['Incoming Requests'] = ['.1.2.826.0.1.1578918.9.5.6.1.2.1'] +HOMESTEAD_OID_DICT['Request Overload Rejections'] = ['.1.2.826.0.1.1578918.9.5.7.1.2.1'] +HOMESTEAD_OID_DICT['MAR Successes'] = ['.1.2.826.0.1.1578918.9.5.10.1.4.1.0.2001'] +HOMESTEAD_OID_DICT['SAR Successes'] = ['.1.2.826.0.1.1578918.9.5.11.1.4.1.0.2001'] +HOMESTEAD_OID_DICT['UAR Successes'] = ['.1.2.826.0.1.1578918.9.5.12.1.4.1.0.2001'] +HOMESTEAD_OID_DICT['LIR Successes'] = ['.1.2.826.0.1.1578918.9.5.13.1.4.1.0.2001'] +HOMESTEAD_OID_DICT['PPR Successes'] = ['.1.2.826.0.1.1578918.9.5.14.1.4.1.0.2001'] +HOMESTEAD_OID_DICT['RTR Successes'] = ['.1.2.826.0.1.1578918.9.5.15.1.4.1.0.2001'] + +STATS_DICT = {'SPROUT': SPROUT_OID_DICT, + 'HOMESTEAD': HOMESTEAD_OID_DICT, 'RALF': RALF_OID_DICT} + + +def get_top(): + return subprocess.check_output('top -b -n 1 -w 78', shell=True).decode('UTF-8') + + +def get_node_info(): + return subprocess.check_output('clearwater-status', shell=True).decode('UTF-8') + + +def get_df(): + rtn_string = 'Your Disk use is higher than our suggested cap, consider consulting support.\nThe use is displayed below:\n\n' + rtn_string += subprocess.check_output('df', shell=True).decode('UTF-8') + return rtn_string + + +def get_etcd_health(): + ''' Returns a string describing the etcd health for the infokeycontroller + ''' + etcd_health = subprocess.check_output( + 'clearwater-etcdctl cluster-health', shell=True).decode('UTF-8').split('\n') + member_list = subprocess.check_output( + 'sudo clearwater-etcdctl member list', shell=True).decode('UTF-8').split('\n') + rtn_string = 'Please see the cluster health stats below:\n' + mapping = {} + for line in member_list: + match_members = re.search('(?P.*?): name=(?P.*?) ', line) + if match_members: + mapping[match_members.group('hash')] = match_members.group('name') + for line in etcd_health: + match = re.search('member (?P.*?) ', line) + if match: + line = re.sub('member (?P.*?) ', 'member ' + + mapping[match.group('hash')] + ' ', line) + line = line.split() + line = ' '.join(line[:4]) + line = '\n' + line + rtn_string += line.strip(':') + return rtn_string + + +def get_cluster_state(): + return subprocess.check_output('/usr/share/clearwater/clearwater-cluster-manager/scripts/check_cluster_state', shell=True).decode('UTF-8') + + +class InfoKeyController(object): + def __init__(self): + self.error_dict = {'CPU USE': get_top, + 'DISK USE': get_df, + 'ETCD CLUSTER': get_etcd_health, + 'MEMCACHED CLUSTER': get_cluster_state, + 'CHRONOS CLUSTER': get_cluster_state, + 'CASSANDRA CLUSTER': get_cluster_state, + 'NODE': get_node_info} + self.key_list = [] + + def update_list(self, error_key_list): + self.key_list = error_key_list + + def get_response(self, error_id): + return self.error_dict[self.key_list[error_id]]() + + def has_element(self, num): + return num >= 0 and num < len(self.key_list) + + +if __name__ == '__main__': + # User MUST be root + if os.geteuid() != 0: + sys.exit('Error, tool must be run as root') + os.environ['NCURSES_NO_UTF8_ACS'] = '1' + logging.basicConfig(level=logging.DEBUG, filename='node_health_tool.log', + filemode='w', format='%(levelname)s-%(asctime)s- %(message)s') + register = InfoKeyController() + # Checking the current node type + try: + for name in NODE_LIST: + version_information = subprocess.check_output( + 'clearwater-version', shell=True).decode('UTF-8') + if name in version_information: + node_name = name + break + logger.info('Node name is %s', node_name) + version_information = version_information.split() + for element in version_information: + if element == node_name: + current_index = version_information.index(element) + node_version = 'v' + version_information[current_index + 1] + except: + version_information = subprocess.check_output( + 'ls /usr/share/clearwater/bin/', shell=True).decode('UTF-8') + for name in NODE_LIST: + if name in version_information: + node_name = name + break + logger.info('Node name is %s', node_name) + node_version = "Project Clearwater" + # Selecting the correct OID dictionary for the statistics table: + stats_dictionary = STATS_DICT[node_name.upper()] + + if node_name != 'ralf': + num_stats = len(stats_dictionary) + else: + num_stats = 2 + + # setting up the fetchers, checkers and viewers + logger.debug('viewer initialised with:\nnode_name=%s\nnode_version=%s\nnum_stats=%s', node_name, node_version, num_stats) + viewer = viewers.Viewer(node_name, node_version, num_stats, register) + viewer.draw() + + cluster_check = checkers.ClusterChecker( + node_name, node_version, viewer.health_group) + node_check = checkers.NodeChecker( + node_name, node_version, viewer.health_group, cluster_check) + first_checker = checkers.UsageChecker( + node_name, node_version, viewer.health_group, node_check) + check_runner = checkers.CheckerController( + first_checker, viewer.health_group) + + FETCHER = {'sprout': fetchers.StatFetcher, + 'homestead': fetchers.StatFetcher, + 'ralf': fetchers.RalfFetcher} + FETCHER_ARGS = {'sprout': [viewer.statistic_window, TIMER_DEFAULT, stats_dictionary], + 'homestead': [viewer.statistic_window, TIMER_DEFAULT, stats_dictionary], + 'ralf': [viewer.statistic_window, TIMER_DEFAULT]} + + fetcher = FETCHER[node_name](*FETCHER_ARGS[node_name]) + + # Starting the two window threads + checking_thread = threading.Thread( + target=check_runner.run, daemon=True, name='checking thread') + stats_thread = threading.Thread( + target=fetcher.run, daemon=True, name='stat fetching thread') + checking_thread.start() + stats_thread.start() + + timer = TIMER_DEFAULT + viewer.flush_input() + + # Input handling + while True: + usr_input = viewer.get_char() + # Exit statement + logger.debug('Key pressed: ' + usr_input) + if usr_input == 'q': + break + # Input for refresh timer + elif usr_input == '+': + timer = viewer.increase_timer(timer) + fetcher.update_timer(timer) + elif usr_input == '-': + timer = viewer.decrease_timer(timer) + fetcher.update_timer(timer) + + # Scroll control + elif usr_input == 'j': + viewer.scroll_down() + elif usr_input == 'k': + viewer.scroll_up() + elif usr_input == 'm': + viewer.scroll_up_info_screen() + elif usr_input == 'n': + viewer.scroll_down_info_screen() + # Handling resize events + elif usr_input == 'key_resize': + viewer.handle_resize() + # Used to go back from the info screen + elif usr_input == 'b': + check_runner.resurrect() + fetcher.resurrect() + viewer.generate_initial_screen() + # Used to take user interaction to errors + elif usr_input.isnumeric() and register.has_element(int(usr_input)): + check_runner.kill() + fetcher.kill() + info_string = register.get_response(int(usr_input)) + viewer.generate_info_screen(info_string) + + viewer.end() diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py b/clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py new file mode 100644 index 00000000..407acf15 --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +'''SNMP tools classes to be used as an alternative to the PySNMP library''' +import logging +import os +import subprocess + + +logger = logging.getLogger(__name__) + + +class SNMP(object): + def __init__(self, community, host): + self.community = community + self.host = host + + def get(self, oid): + raise NotImplementedError("An abstract method was called.") + + +class SubProcessSNMP(SNMP): + def get(self, oid): + logger.debug('performing snmpget operation on: %s', oid) + with open(os.devnull, 'w') as FNULL: + rtn_string = subprocess.check_output('snmpget -v2c -c ' + ' '.join([self.community, self.host, oid]), stderr=FNULL, shell=True).decode('UTF-8') + logger.debug('splitting returned information: %s', rtn_string) + rtn_value = rtn_string.split(': ')[1] + rtn_value = rtn_value.strip() + if rtn_value.isnumeric(): + rtn_value = float(rtn_value) + return rtn_value diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/viewers.py b/clearwater-infrastructure/usr/bin/health-tool_source/viewers.py new file mode 100644 index 00000000..ff3d60c2 --- /dev/null +++ b/clearwater-infrastructure/usr/bin/health-tool_source/viewers.py @@ -0,0 +1,803 @@ +''' Viewer related objects + + Colour pair guide by pair index: + 1 - red on black + 2 - green on black + 3 - cyan on black + 4 - yellow on black +''' +import curses +from curses.textpad import rectangle +import logging +from collections import OrderedDict + +# Initialise logger +logger = logging.getLogger(__name__) + +# Default variables +TIMER_DEFAULT = 5 +STATS_DISPLAYED = 13 +STAT_LIST = [] +for i in range(26): + STAT_LIST.append(('Name' + str(i), str(i))) +NUM_STATS = len(STAT_LIST) + + +class ScrollBar(object): + '''A class for scroll bars + ''' + + def __init__(self, upper_left_y, upper_left_x, length, num_scrolls, scroll_up='k', scroll_down='j', scroll_level=0): + '''Args: + upper_left_y: The y coordinate of the upper left corner of the scroll bar + upper_left_x: The x coordinate of the upper left corner of the scroll bar + length: The length of the scroll bar (This MUST be greater than or equal to 2) + num_scrolls: The maximum number of times a user can scroll in either direction (i.e. the number of lines off-screen) + scroll_level: The location of the scroll bump inside the bar, generally 0 to start at the top. + scroll_up: The letter used to scroll up + scroll_down: The letter used to scroll down + ''' + # The (- 1) is to compensate for scroll_level counting from 0 + self.max_row = length - 1 + self.scroll_max = num_scrolls + logger.debug('scroll max set to: %s', self.scroll_max) + self.upper_left_y = upper_left_y + self.upper_left_x = upper_left_x + self.up = scroll_up + self.scroll_level = scroll_level + self.down = scroll_down + self.scroll_window = curses.newwin(length, + 2, + upper_left_y, + upper_left_x) + logger.debug('scrolling bar to level: %s', scroll_level) + self.scroll_to(scroll_level) + + def show_up_arrow(self): + ''' Reveals the up arrow and key to the UI + ''' + self.scroll_window.addstr(0, 0, '↑') + self.scroll_window.addstr(self.up, curses.color_pair(3)) + self.scroll_window.refresh() + + def show_down_arrow(self): + ''' Reveals the down arrow and key to the UI + ''' + self.scroll_window.addstr(self.max_row, 0, '↓') + # This try-except is due to curses pushing the cursor off screen and erroring + try: + self.scroll_window.addstr( + self.max_row, 1, self.down, curses.color_pair(3)) + except curses.error: + logging.debug('curses pushed cursor out of the window') + self.scroll_window.refresh() + + def remove_down_arrow(self): + ''' Removes the down arrow and key from the UI + ''' + # This try-except is due to curses pushing the cursor off screen and erroring + try: + self.scroll_window.addstr(self.max_row, 0, ' ') + except curses.error: + logging.debug('curses pushed cursor out of the window') + self.scroll_window.refresh() + + def remove_up_arrow(self): + ''' Removes the up arrow and key from the UI + ''' + self.scroll_window.addstr(0, 0, ' ') + self.scroll_window.refresh() + + def scroll_to(self, scroll_level): + ''' Scrolls the bar to a specific 'level', which can be visualised as the scroll + bars position within its bounds on a standard GUI + + Args: + scroll_level: The level to which the bar is to be set + ''' + if scroll_level > 0 and scroll_level < self.scroll_max: + self.show_up_arrow() + self.show_down_arrow() + elif scroll_level > 0 and scroll_level == self.scroll_max: + self.show_up_arrow() + self.remove_down_arrow() + elif scroll_level == 0 and scroll_level < self.scroll_max: + logger.info('initialising the scroll window') + self.show_down_arrow() + self.remove_up_arrow() + + +class ScrollableWindow(object): + ''' This is a class for a window which contains some rows of text and can be scrolled within if there + are more rows of text than space in it's display area + + This works on the assumption that the command will output in a standard 80x25 size + ''' + + def __init__(self, stdscr, upper_left_y, upper_left_x, lower_right_y, lower_right_x): + ''' Args: + stdscr: The screen in which this window is to be set + upper_left_y: The upper left y coordinate of this window with respect to the stdscr + upper_left_x: The upper left x coordinate of this window with respect to the stdscr + lower_right_y: The upper left y coordinate of this window with respect to the stdscr + lower_right_x: The upper left x coordinate of this window with respect to the stdscr + ''' + self.text_space_width = lower_right_x - upper_left_x + 1 + self.displayed_rows = lower_right_y - upper_left_y + self.bottom_line = curses.newwin(1, + self.text_space_width, + lower_right_y, + upper_left_x) + self.scroll = 0 + self.last_data = None + self.upper_left_y = upper_left_y + self.upper_left_x = upper_left_x + self.lower_right_y = lower_right_y + self.lower_right_x = lower_right_x + self.stdscr = stdscr + self.rows = 1 + + def update_bottom_line(self): + ''' Reprints the bottom line with a message telling the user to press b to go back + ''' + self.bottom_line.addstr(0, + 0, + 'press to return to the previous screen') + self.bottom_line.addstr(0, len('press '), 'b', curses.color_pair(3)) + self.bottom_line.refresh() + + def update_pad(self, data): + ''' Updates the pad and bottom line, essentially re-fills the pad and refreshes + all items within the ScrollableWindow + ''' + self.rows = len(data.split('\n')) + self.scroll = 0 + logger.debug('Number of rows in ScrollableWindow ' + str(self.rows)) + self.pad = curses.newpad(self.rows, self.text_space_width) + logger.debug(data) + self.pad.addstr(0, 0, data) + self.last_data = data + self.stdscr.refresh() + self.scroll_bar = ScrollBar(self.upper_left_y, + self.lower_right_x - 1, + self.displayed_rows, + self.rows - self.displayed_rows, + scroll_up='m', + scroll_down='n') + self.pad.refresh(self.scroll, + 0, + self.upper_left_y, + self.upper_left_x, + self.lower_right_y - 1, + self.lower_right_x - 2) + self.update_bottom_line() + + def scroll_up(self): + ''' Scrolls the ScrollableWindow upwards, affecting both the scroll bar and the pad, not the bottom line + (bottom line just updates) + ''' + if self.scroll > 0: + logger.info('scroll value is: %s', self.scroll) + self.scroll -= 1 + logger.info('scroll value changed to: %s', self.scroll) + self.scroll_bar.scroll_to(self.scroll) + logger.info('sent info to scroll bar') + self.stdscr.refresh() + self.pad.refresh(self.scroll, + 0, + self.upper_left_y, + self.upper_left_x, + self.lower_right_y - 1, + self.lower_right_x - 2) + self.update_bottom_line() + + def scroll_down(self): + ''' Scrolls the ScrollableWindow downwards, affecting the scroll bar and the pad, not the bottom line + (bottom line just updates) + ''' + logger.info('scrolling down') + if self.scroll < self.rows - self.displayed_rows: + logger.info('scroll value is: %s', self.scroll) + self.scroll += 1 + logger.info('scroll value changed to: %s', self.scroll) + self.scroll_bar.scroll_to(self.scroll) + logger.debug('sent info to scroll bar') + self.stdscr.refresh() + self.pad.refresh(self.scroll, + 0, + self.upper_left_y, + self.upper_left_x, + self.lower_right_y - 1, + self.lower_right_x - 2) + self.update_bottom_line() + + def refresh(self): + if self.last_data: + self.update_pad(self.last_data) + + +class TableWindow(object): + ''' This is a class for a TableWindow, made up of 4 objects. + 1- Headings window, located at the top of the TableWindow, under the border, + is one row in height and the full internal width of the window. + 2- Table pad, this is a table located beneath the Headings window, and is + the main occupier of space. It is 2 columns thinner than the Headings window + to allow room for a scroll bar + 3- Scroll bar, this is a scroll bar located beside the Table pad to ensure that + the user is aware of where in the table they are + 4- Timer Window, this is a window to hold the information regarding the refresh + timer. It is located beneath the Table pad and is the entire width of the window + + A visual representation of this is below: + --------------------------------- + |1111111111111111111111111111111| + |2222222222222222222222222222233| + |2222222222222222222222222222233| + |2222222222222222222222222222233| + |4444444444444444444444444444444| + --------------------------------- + ''' + + def __init__(self, stdscr, upper_left_y, upper_left_x, lower_right_y, lower_right_x, rows, name_width, default_timer=5): + ''' Args: + stdscr: The screen that the window is nested in + upper_left_y: The y coordinate of the upper left corner of the rectangle + upper_left_x: The x coordinate of the upper left corner of the rectangle + lower_right_y: The y coordinate of the lower right corner of the rectangle + lower_right_x: The x coordinate of the lower right corner of the rectangle + rows: The number of rows to have in the table in total + displayed_rows: The number of rows to display at any one time + name_width: The width of the name column + default_timer: The timer default + ''' + self.text_space_width = lower_right_x - upper_left_x - 1 + self.table_pad = curses.newpad(rows, self.text_space_width) + displayed_rows = lower_right_y - upper_left_y - 3 + self.scroll_bar = ScrollBar(upper_left_y + 2, + lower_right_x - 2, + displayed_rows, + rows - displayed_rows) + self.name_width = name_width + self.scroll = 0 + self.last_data = None + self.rows = rows + self.upper_left_y = upper_left_y + self.upper_left_x = upper_left_x + self.lower_right_y = lower_right_y + self.lower_right_x = lower_right_x + self.default_timer = default_timer + self.displayed_rows = displayed_rows + self.stdscr = stdscr + self.statistics = [] + self.refresh() + + def refresh(self): + ''' Refreshes the TableWindow, repainting the elements nested on top of it''' + logger.debug('generating a rectangle with dimensions\nULY: %s\nULX: %s\nLRY: %s\nLRX: %s', self.upper_left_y, self.upper_left_x, self.lower_right_y, self.lower_right_x) + rectangle(self.stdscr, + self.upper_left_y, + self.upper_left_x, + self.lower_right_y, + self.lower_right_x) + self.stdscr.refresh() + self.headings_window = curses.newwin(1, + self.text_space_width, + self.upper_left_y + 1, + self.upper_left_x + 1) + + for i in range(self.text_space_width - 1): + self.headings_window.addstr(' ', curses.A_REVERSE) + + self.headings_window.addstr(0, 0, 'Statistic Name', curses.A_REVERSE) + self.headings_window.addstr(0, + self.name_width, + 'Statistic Value', + curses.A_REVERSE) + self.headings_window.refresh() + self.timer_window = curses.newwin(1, + self.text_space_width, + self.lower_right_y - 1, + self.upper_left_x + 1) + + self.timer_window.addstr(0, + 0, + 'press or to change the refresh interval:') + self.timer_window.addstr(0, 6, '+', curses.color_pair(3)) + self.timer_window.addstr(0, 11, '-', curses.color_pair(3)) + self.update_timer(self.default_timer) + + self.table_pad.refresh(self.scroll, + 0, + self.upper_left_y + 2, + self.upper_left_x + 1, + self.lower_right_y - 2, + self.lower_right_x - 3) + self.timer_window.refresh() + self.scroll_bar = ScrollBar(self.upper_left_y + 2, + self.lower_right_x - 2, + self.displayed_rows, + self.rows - self.displayed_rows, + scroll_level=self.scroll) + + def update(self): + self.last_statistics = self.statistics + self.write_to_table(self.statistics) + + def increase_timer(self, timer, time=5): + if timer < 1000: + timer += 5 + self.update_timer(timer) + return timer + + def decrease_timer(self, timer, time=5): + if timer > 5: + timer -= 5 + self.update_timer(timer) + return timer + + def write_to_table(self, data): + ''' Args: + data: The expected Args is a list of tuples in the format [(Name, Value), (Name, Value),...] + ''' + self.last_data = data + i = 0 + self.table_pad.clear() + if len(data) > self.rows: + logger.error('Data passed to table has more entries than rows available') + while i < self.rows: + self.table_pad.addstr(i, 0, data[i][0]) + self.table_pad.addstr(i, self.name_width, str(data[i][1])) + i += 1 + self.table_pad.refresh(self.scroll, + 0, + self.upper_left_y + 2, + self.upper_left_x + 1, + self.lower_right_y - 2, + self.lower_right_x - 3) + + def update_timer(self, timer): + ''' Args: + timer: The expected Args is an integer + ''' + self.timer_window.addstr(0, 45, ' ') + self.timer_window.addstr(0, 45, str(timer), curses.A_REVERSE) + self.timer_window.refresh() + + def scroll_up(self): + if self.scroll > 0: + self.scroll -= 1 + self.scroll_bar.scroll_to(self.scroll) + self.table_pad.refresh(self.scroll, 0, + self.upper_left_y + 2, + self.upper_left_x + 1, + self.lower_right_y - 2, + self.lower_right_x - 3) + + def scroll_down(self): + logger.debug('scrolling down in scrollable window') + if self.scroll < self.rows - self.displayed_rows: + self.scroll += 1 + self.scroll_bar.scroll_to(self.scroll) + self.table_pad.refresh(self.scroll, 0, + self.upper_left_y + 2, + self.upper_left_x + 1, + self.lower_right_y - 2, + self.lower_right_x - 3) + + def add_statistics(self, data): + self.statistics = data + + def display_loading_message(self): + self.table_pad.addstr(0, 0, 'Loading...') + self.table_pad.refresh(0, + 0, + self.upper_left_y + 2, + self.upper_left_x + 1, + self.lower_right_y - 2, + self.lower_right_x - 3) + + def remove_loading_message(self): + self.table_pad.addstr(0, 0, ' ') + + +class StatusWindow(object): + ''' This is a class for a StatusWindow, a window which will display health status + in colours varying by the severity of the condition of the element. It will + also print the number beside the erroring checks and use these numbers to tell + a registration object which are erroring. + ''' + def __init__(self, stdscr, upper_left_y, upper_left_x, lower_right_y, lower_right_x, register, status='HEALTH STATUS'): + ''' Args: + + stdscr: The screen in which the window is nested + upper_left_y: The y coordinate of the upper left corner of the rectangle + upper_left_x: The x coordinate of the upper left corner of the rectangle + lower_right_y: The y coordinate of the lower right corner of the rectangle + lower_right_x: The x coordinate of the lower right corner of the rectangle + register: The list registration object for + status: The title for the window, typically 'HEALTH STATUS' + ''' + rectangle(stdscr, upper_left_y, upper_left_x, + lower_right_y, lower_right_x) + stdscr.refresh() + self.rows = lower_right_y - upper_left_y - 1 + self.columns = lower_right_x - upper_left_x - 1 + self.status_window = curses.newwin(self.rows, + self.columns, + upper_left_y + 1, + upper_left_x + 1) + self.last_data = None + self.stdscr = stdscr + self.status = status + self.upper_left_y = upper_left_y + self.upper_left_x = upper_left_x + self.lower_right_x = lower_right_x + self.lower_right_y = lower_right_y + self.refresh() + self.status_window.refresh() + self.register = register + + def display_loading_message(self): + self.status_window.addstr(2, 0, 'Loading...') + self.status_window.refresh() + + def remove_loading_message(self): + self.status_window.addstr(2, 0, ' ') + + def write_status(self, data): + ''' A method to write the status to the status window, and then refresh to display this to + a user. + Args: + + data: Args should be in the form of a dictionary, in the order + {Name: Health_stat, ...] where Health_stat + is an string from either 'ERROR', 'WARNING', 'GOOD'. + ''' + self.last_data = data + self.status_window.move(2, 0) + i = 0 + failures = [] + # Error_index is the number a user will press to see that particular + # item's error message + error_index = 0 + + for i, element in enumerate(data): + if data[element] == 'GOOD': + colour = curses.color_pair(1) + index = ' ' + else: + if data[element] == 'WARNING': + colour = curses.color_pair(4) + else: + colour = curses.color_pair(2) + index = error_index + error_index += 1 + failures.append(element) + + self.status_window.addstr(2 + i, + 0, + str(index), + curses.color_pair(3)) + self.status_window.addstr(2 + i, 1, element, colour) + self.register.update_list(failures) + self.status_window.refresh() + + def refresh(self): + rectangle(self.stdscr, + self.upper_left_y, + self.upper_left_x, + self.lower_right_y, + self.lower_right_x) + self.stdscr.refresh() + self.status_window = curses.newwin(self.rows, + self.columns, + self.upper_left_y + 1, + self.upper_left_x + 1) + self.status_window.addstr(0, 0, self.status + '\n', curses.A_BOLD) + + for i in range(self.columns): + self.status_window.addstr('-', curses.A_BOLD) + + self.status_window.addstr(self.rows - 1, 0, 'Press # for info') + if self.last_data: + self.write_status(self.last_data) + + +class InformationBox(object): + ''' A class for an information box, displayed at the top of the screen + which will show a maximum number of error messages for a user + ''' + def __init__(self, stdscr, upper_left_y, upper_left_x, lower_right_y, lower_right_x): + ''' Args: + stdscr: The screen in which the information box is nested + upper_left_y: The upper left y coordinate of the box + upper_left_x: The upper left x coordinate of the box + lower_right_y: The lower right y coordinate of the box + lower_right_x: The lower right y coordinate of the box + ''' + self.upper_left_y = upper_left_y + self.upper_left_x = upper_left_x + self.lower_right_y = lower_right_y + self.lower_right_x = lower_right_x + self.stdscr = stdscr + self.text_space_width = lower_right_x - upper_left_x - 1 + self.rows = lower_right_y - upper_left_y - 1 + self.max_row = self.rows - 1 + self.last_info = None + self.refresh() + + def refresh(self): + rectangle(self.stdscr, self.upper_left_y, self.upper_left_x, + self.lower_right_y, self.lower_right_x) + self.stdscr.refresh() + self.information_window = curses.newwin(self.rows, + self.text_space_width, + self.upper_left_y + 1, + self.upper_left_x + 1) + self.lay_window_base() + if self.last_info: + self.write_information(self.last_info) + self.information_window.refresh() + + def lay_window_base(self): + self.information_window.addstr(self.max_row, + 0, + 'Press to quit') + self.information_window.addstr(self.max_row, + 6, + 'q', + curses.color_pair(3)) + + def write_information(self, data, name_width=25): + ''' Args: + + data: Input information given by a dictionary of lists of tuples in the format + {'WARNING': [(Name, info_string),...], 'ERROR': [(Name, info_string)]} + name_width: the width of the name column of the table + ''' + self.last_info = data + logger.debug('Information box passed the following data: %s', data) + i = 0 + self.information_window.clear() + + # Printing the Errors + for element in data['ERROR']: + if i == 5: + break + self.information_window.addstr( + i, 0, element, curses.color_pair(2)) + self.information_window.addstr(' ERROR:', curses.color_pair(2)) + self.information_window.addstr(i, name_width, '!!!') + self.information_window.addstr(data['ERROR'][element]) + self.information_window.addstr('!!!') + i += 1 + + # Printing the Warnings + for element in data['WARNING']: + if i == 5: + break + self.information_window.addstr( + i, 0, element, curses.color_pair(4)) + self.information_window.addstr(' WARNING:', curses.color_pair(4)) + self.information_window.addstr(i, name_width, data['WARNING'][element]) + i += 1 + self.lay_window_base() + self.information_window.refresh() + + +class HealthGroup(object): + ''' This is a wrapper to cover a StatusWindow and an InformationBox + it is useful for keeping the processes controlling the health information + and the statistics separated. + ''' + def __init__(self, health_window, information_box): + self.health_window = health_window + self.information_box = information_box + self.wipe_info() + self.health_dict = OrderedDict() + + def update(self): + self.last_health = self.health_dict + self.last_info = self.information + self.health_window.write_status(self.health_dict) + self.information_box.write_information(self.information) + + def add_warning_info(self, name, message): + self.information['WARNING'][name] = message + + def add_error_info(self, name, message): + self.information['ERROR'][name] = message + + def wipe_info(self): + error_dict = OrderedDict() + warning_dict = OrderedDict() + self.information = {'ERROR': error_dict, 'WARNING': warning_dict} + + def health_addition(self, name, value): + self.health_dict[name] = value + + def display_loading_message(self): + self.health_window.display_loading_message() + + def remove_loading_message(self): + self.health_window.remove_loading_message() + + +class Viewer(object): + ''' A class for a viewer which combines the elements of this doc into a neat and useful viewer. + ''' + def __init__(self, node_name, node_version, num_stats, register): + ''' Colour guide by pair index: + 1 - red on black + 2 - green on black + 3 - cyan on black + 4 - yellow on black + Args: + node_name: The name of the node to be displayed + node_version: The version of the node to be displayed + num_stats: The number of stats to be displayed in the statistics table + register: The register object which will coordinate the passing of + information regarding error-reactions from the viewer to the main script + + ''' + # Initial main window settings + self.register = register + self.screen = curses.initscr() + curses.noecho() + curses.cbreak() + curses.start_color() + curses.use_default_colors() + curses.resizeterm(25, 80) + self.screen.border(0) + curses.curs_set(0) + self.current_screen = 'MAIN' + + # Initialise colours, for guide see above + curses.init_pair(1, curses.COLOR_GREEN, curses.COLOR_BLACK) + curses.init_pair(2, curses.COLOR_RED, curses.COLOR_BLACK) + curses.init_pair(3, curses.COLOR_CYAN, curses.COLOR_BLACK) + curses.init_pair(4, curses.COLOR_YELLOW, curses.COLOR_BLACK) + + self.initial_timer = 5 + self.num_stats = num_stats + self.node_name = node_name + self.node_version = node_version + + def update_node_info(self): + self.screen.addstr(1, 20, 'Node: ' + self.node_name, curses.A_BOLD) + self.screen.addstr(' ' + self.node_version, curses.A_BOLD) + self.screen.refresh() + + def draw(self): + ''' Creates the window for the user, drawing all of the elements + ''' + self.update_node_info() + self.statistic_window = TableWindow(self.screen, + 10, + 1, + 23, + 57, + self.num_stats, + 35, + self.initial_timer) + self.health_window = StatusWindow(self.screen, + 10, + 59, + 23, + 78, + self.register) + self.information_window = InformationBox(self.screen, 2, 1, 9, 78) + self.health_group = HealthGroup(self.health_window, + self.information_window) + self.info_screen = ScrollableWindow(self.screen, 0, 0, 24, 79) + + def update_statistics(self): + self.last_statistics = self.statistics + self.statistic_window.write_to_table(self.statistics) + + def update(self): + self.update_statistics() + self.health_group.update() + + def end(self): + ''' Ends the viewer cleanly to return the the Terminal + ''' + curses.nocbreak() + curses.echo() + curses.endwin() + curses.curs_set(1) + + def increase_timer(self, timer, time=5): + if timer < 1000: + timer += 5 + self.statistic_window.update_timer(timer) + return timer + + def decrease_timer(self, timer, time=5): + if timer > 5: + timer -= 5 + self.statistic_window.update_timer(timer) + return timer + + def get_char(self): + return self.screen.getkey().lower() + + def handle_resize(self): + ''' Handles a resize event by refreshing the screen and, if the screen is smaller than the standard size, + displays an error message + ''' + logger.debug('Received a KEY_RESIZE character') + if self.current_screen == 'INFORMATION': + self.refresh_info_screen() + else: + self.refresh_initial_screen() + (y, x) = self.screen.getmaxyx() + logger.debug('screen dimensions are %s, %s', y, x) + if y < 25 or x < 80: + while y < 25 or x < 80: + (y, x) = self.screen.getmaxyx() + self.screen.clear() + curses.resizeterm(y, x) + self.screen.border(0) + self.screen.addstr( + 0, 0, 'Please do not resize your screen to smaller than the borders while using the tool, resize to continue\n') + self.screen.refresh() + if self.current_screen == 'INFORMATION': + self.generate_info_screen(self.current_information_string) + else: + self.generate_initial_screen() + curses.flushinp() + + def refresh_initial_screen(self): + ''' Refreshes the 'main' screen''' + self.screen.refresh() + self.statistic_window.refresh() + self.health_window.refresh() + self.information_window.refresh() + + def refresh_info_screen(self): + ''' Refreshes the additional information screen''' + self.info_screen.refresh() + + def generate_initial_screen(self): + '''Re-creates the 'main' screen''' + self.current_screen = 'MAIN' + self.screen.erase() + curses.resizeterm(25, 80) + self.screen.border(0) + self.update_node_info() + self.statistic_window.refresh() + self.health_window.refresh() + self.information_window.refresh() + curses.flushinp() + + def generate_info_screen(self, information_string): + ''' Creates the additional information screen displayed if a user wishes + to find out more about an error message. + + Args: + information_string: This is the string to be displayed on the + additional information screen + ''' + self.current_screen = 'INFORMATION' + self.current_information_string = information_string + self.screen.erase() + curses.resizeterm(25, 80) + self.info_screen.update_pad(information_string) + self.screen.refresh() + curses.flushinp() + + def scroll_down(self): + self.statistic_window.scroll_down() + + def scroll_up(self): + self.statistic_window.scroll_up() + + def scroll_down_info_screen(self): + self.info_screen.scroll_down() + + def scroll_up_info_screen(self): + self.info_screen.scroll_up() + + def flush_input(self): + curses.flushinp() From 8b1799f6f52f6c4523ea9aeffe953f2b9260b217 Mon Sep 17 00:00:00 2001 From: Ross Paterson Date: Fri, 26 Aug 2016 09:51:02 +0100 Subject: [PATCH 2/3] updated directory location for source of Node Health Tool --- clearwater-infrastructure/usr/bin/health-tool | 3 +-- .../clearwater/health_tool}/checkers.py | 0 .../clearwater/health_tool}/fetchers.py | 0 .../health-tool_source => share/clearwater/health_tool}/run.py | 0 .../clearwater/health_tool}/snmpy.py | 0 .../clearwater/health_tool}/viewers.py | 0 6 files changed, 1 insertion(+), 2 deletions(-) rename clearwater-infrastructure/usr/{bin/health-tool_source => share/clearwater/health_tool}/checkers.py (100%) rename clearwater-infrastructure/usr/{bin/health-tool_source => share/clearwater/health_tool}/fetchers.py (100%) rename clearwater-infrastructure/usr/{bin/health-tool_source => share/clearwater/health_tool}/run.py (100%) rename clearwater-infrastructure/usr/{bin/health-tool_source => share/clearwater/health_tool}/snmpy.py (100%) rename clearwater-infrastructure/usr/{bin/health-tool_source => share/clearwater/health_tool}/viewers.py (100%) diff --git a/clearwater-infrastructure/usr/bin/health-tool b/clearwater-infrastructure/usr/bin/health-tool index 4cc91571..ceb83c4c 100644 --- a/clearwater-infrastructure/usr/bin/health-tool +++ b/clearwater-infrastructure/usr/bin/health-tool @@ -1,3 +1,2 @@ #!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; -$DIR/run.py \ No newline at end of file +/usr/share/clearwater/health_tool/run.py; \ No newline at end of file diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/checkers.py b/clearwater-infrastructure/usr/share/clearwater/health_tool/checkers.py similarity index 100% rename from clearwater-infrastructure/usr/bin/health-tool_source/checkers.py rename to clearwater-infrastructure/usr/share/clearwater/health_tool/checkers.py diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py b/clearwater-infrastructure/usr/share/clearwater/health_tool/fetchers.py similarity index 100% rename from clearwater-infrastructure/usr/bin/health-tool_source/fetchers.py rename to clearwater-infrastructure/usr/share/clearwater/health_tool/fetchers.py diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/run.py b/clearwater-infrastructure/usr/share/clearwater/health_tool/run.py similarity index 100% rename from clearwater-infrastructure/usr/bin/health-tool_source/run.py rename to clearwater-infrastructure/usr/share/clearwater/health_tool/run.py diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py b/clearwater-infrastructure/usr/share/clearwater/health_tool/snmpy.py similarity index 100% rename from clearwater-infrastructure/usr/bin/health-tool_source/snmpy.py rename to clearwater-infrastructure/usr/share/clearwater/health_tool/snmpy.py diff --git a/clearwater-infrastructure/usr/bin/health-tool_source/viewers.py b/clearwater-infrastructure/usr/share/clearwater/health_tool/viewers.py similarity index 100% rename from clearwater-infrastructure/usr/bin/health-tool_source/viewers.py rename to clearwater-infrastructure/usr/share/clearwater/health_tool/viewers.py From 6efce1de3d9b7ffbead253b736a971a2f6a7447a Mon Sep 17 00:00:00 2001 From: Ross Paterson Date: Fri, 26 Aug 2016 09:57:31 +0100 Subject: [PATCH 3/3] Added python 3.4.3 to dependencies --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 7fa13a9d..5824db5a 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Homepage: http://projectclearwater.org/ Package: clearwater-infrastructure Architecture: any -Depends: dnsmasq, ntp, python2.7, python-setuptools, gnutls-bin, libzmq3 +Depends: dnsmasq, ntp, python2.7, python-setuptools, gnutls-bin, libzmq3, python3.4.3 Suggests: clearwater-auto-config, clearwater-auto-upgrade Recommends: clearwater-diags-monitor Description: Common infrastructure for all Clearwater servers