diff --git a/rollastic/node.py b/rollastic/node.py index 11ff7cb..9f47f07 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -19,7 +19,7 @@ class Node(dict): Represents a cluster node. ''' - def __init__(self, cluster, node_id=None): + def __init__(self, cluster, node_id=None, bad_result_maximum=10): ''' Init @@ -30,6 +30,9 @@ def __init__(self, cluster, node_id=None): ''' self.cluster = cluster self.node_id = node_id + self.bad_result_maximum = bad_result_maximum + self.bad_result_count = 0 + if self.node_id: self.populate() @@ -53,15 +56,26 @@ def populate(self): ''' self.clear() - info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) - if not info: - _LOG.warning('Bad result for node info. node_id=%s info=%s', self.node_id, info) - self.update(info) + try: + info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) + self.update(info) - stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) - if not stats: - _LOG.warning('Bad result for node stats. node_id=%s stats=%s', self.node_id, stats) - self.update(stats) + stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) + self.update(stats) + + if not stats or info: + raise Exception("Failed to retrieve values from node. node_id=%s info=%s stats=%s", + self.node_id, info, stats) + + # reset the counter to zero because we only care about tracking consecutive bad results from the node + self.bad_result_count = 0 + except Exception as e: + self.bad_result_count += 1 + _LOG.warning("{0} (failure {1} of {2})".format(str(e), self.bad_result_count, self.bad_result_maximum)) + + if self.bad_result_count > self.bad_result_maximum: + raise Exception("Reached max failure count when retrieving information from node. node_id=%s", + self.node_id) @property def name(self):