From 4bbc15b2b97d01b695601d63b248dd91601ffeb3 Mon Sep 17 00:00:00 2001 From: Alexander Alber Date: Tue, 12 Apr 2016 16:06:46 -0400 Subject: [PATCH 1/5] Raise exception when we can't get get stats or info from a node --- rollastic/node.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rollastic/node.py b/rollastic/node.py index 11ff7cb..281d088 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -55,12 +55,16 @@ def populate(self): info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) if not info: - _LOG.warning('Bad result for node info. node_id=%s info=%s', self.node_id, info) + msg = "Bad result for node info. node_id={} info={}".format(self.node_id, info) + _LOG.error(msg) + raise Exception(msg) self.update(info) stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) if not stats: - _LOG.warning('Bad result for node stats. node_id=%s stats=%s', self.node_id, stats) + msg = "Bad result for node stats. node_id={} stats={}".format(self.node_id, stats) + _LOG.error(msg) + raise Exception(msg) self.update(stats) @property From 951f6404ee1846ac7df5443ee8a2931ad52f7245 Mon Sep 17 00:00:00 2001 From: Alexander Alber Date: Tue, 12 Apr 2016 16:18:50 -0400 Subject: [PATCH 2/5] Use numbered replacement arguments --- rollastic/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rollastic/node.py b/rollastic/node.py index 281d088..34cafd4 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -55,14 +55,14 @@ def populate(self): info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) if not info: - msg = "Bad result for node info. node_id={} info={}".format(self.node_id, info) + msg = "Bad result for node info. node_id={0} info={1}".format(self.node_id, info) _LOG.error(msg) raise Exception(msg) self.update(info) stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) if not stats: - msg = "Bad result for node stats. node_id={} stats={}".format(self.node_id, stats) + msg = "Bad result for node stats. node_id={0} stats={1}".format(self.node_id, stats) _LOG.error(msg) raise Exception(msg) self.update(stats) From 8ff2d6972a24152a7d854566bb6b6610dd5bf594 Mon Sep 17 00:00:00 2001 From: Alexander Alber Date: Tue, 12 Apr 2016 16:43:47 -0400 Subject: [PATCH 3/5] Keep track of the number of times we fail to populate info from a node. If the number of failures exceeds 10, raise an exception. --- rollastic/node.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/rollastic/node.py b/rollastic/node.py index 34cafd4..35f8005 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -30,6 +30,7 @@ def __init__(self, cluster, node_id=None): ''' self.cluster = cluster self.node_id = node_id + self.badcount = 0 if self.node_id: self.populate() @@ -54,17 +55,26 @@ def populate(self): self.clear() info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) + isbad = False if not info: msg = "Bad result for node info. node_id={0} info={1}".format(self.node_id, info) - _LOG.error(msg) - raise Exception(msg) + _LOG.warning(msg) + isbad = True self.update(info) stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) if not stats: msg = "Bad result for node stats. node_id={0} stats={1}".format(self.node_id, stats) - _LOG.error(msg) - raise Exception(msg) + _LOG.warning(msg) + isbad = True + + # Make sure that we don't infintely wait on nodes that can't get a good result + if isbad: + self.badcount += 1 + if self.badcount > 10: + raise Exception("Too many bad results for while populating node. node_id={0}".format(self.node_id)) + else: + self.badcount = 0 self.update(stats) @property From 1a438b3a9af15e91ff074bf075702bf9e0467a24 Mon Sep 17 00:00:00 2001 From: Alexander Alber Date: Wed, 13 Apr 2016 10:02:43 -0400 Subject: [PATCH 4/5] Handle flow of populate() with an exception instead of a flag --- rollastic/node.py | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/rollastic/node.py b/rollastic/node.py index 35f8005..8395fd2 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -19,7 +19,7 @@ class Node(dict): Represents a cluster node. ''' - def __init__(self, cluster, node_id=None): + def __init__(self, cluster, node_id=None, bad_result_maximum=10): ''' Init @@ -30,7 +30,9 @@ def __init__(self, cluster, node_id=None): ''' self.cluster = cluster self.node_id = node_id - self.badcount = 0 + self.bad_result_maximum = bad_result_maximum + self.bad_result_count = 0 + if self.node_id: self.populate() @@ -54,28 +56,26 @@ def populate(self): ''' self.clear() - info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) - isbad = False - if not info: - msg = "Bad result for node info. node_id={0} info={1}".format(self.node_id, info) - _LOG.warning(msg) - isbad = True - self.update(info) - - stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) - if not stats: - msg = "Bad result for node stats. node_id={0} stats={1}".format(self.node_id, stats) - _LOG.warning(msg) - isbad = True - - # Make sure that we don't infintely wait on nodes that can't get a good result - if isbad: - self.badcount += 1 - if self.badcount > 10: - raise Exception("Too many bad results for while populating node. node_id={0}".format(self.node_id)) - else: - self.badcount = 0 - self.update(stats) + try: + info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {}) + self.update(info) + + stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {}) + self.update(stats) + + if not stats or info: + raise Exception("Failed to retrieve values from node. node_id={0} info={1} stats={2}".format( + self.node_id, info, stats)) + + # reset the counter to zero because we only care about tracking consecutive bad results from the node + self.bad_result_count = 0 + except Exception as e: + self.bad_result_count += 1 + _LOG.warning("{0} (failure {1} of {2}".format(str(e), self.bad_result_count, self.bad_result_maximum)) + + if self.bad_result_count > self.bad_result_maximum: + raise Exception("Reached max failure count when retrieving information from node. node_id={0}".format( + self.node_id)) @property def name(self): From 1e461396f57dd3f697eaa39b2c59ee5b111ad882 Mon Sep 17 00:00:00 2001 From: Alexander Alber Date: Tue, 3 May 2016 13:58:19 -0400 Subject: [PATCH 5/5] Use sprintf formatting over the .format method --- rollastic/node.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rollastic/node.py b/rollastic/node.py index 8395fd2..9f47f07 100644 --- a/rollastic/node.py +++ b/rollastic/node.py @@ -64,18 +64,18 @@ def populate(self): self.update(stats) if not stats or info: - raise Exception("Failed to retrieve values from node. node_id={0} info={1} stats={2}".format( - self.node_id, info, stats)) + raise Exception("Failed to retrieve values from node. node_id=%s info=%s stats=%s", + self.node_id, info, stats) # reset the counter to zero because we only care about tracking consecutive bad results from the node self.bad_result_count = 0 except Exception as e: self.bad_result_count += 1 - _LOG.warning("{0} (failure {1} of {2}".format(str(e), self.bad_result_count, self.bad_result_maximum)) + _LOG.warning("{0} (failure {1} of {2})".format(str(e), self.bad_result_count, self.bad_result_maximum)) if self.bad_result_count > self.bad_result_maximum: - raise Exception("Reached max failure count when retrieving information from node. node_id={0}".format( - self.node_id)) + raise Exception("Reached max failure count when retrieving information from node. node_id=%s", + self.node_id) @property def name(self):