Skip to content

Raise exception when we can't get get stats or info from a node #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bender-the-greatest wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

Raise exception when we can't get get stats or info from a node #5

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4bbc15b
Raise exception when we can't get get stats or info from a node
bender-the-greatest Apr 12, 2016
951f640
Use numbered replacement arguments
bender-the-greatest Apr 12, 2016
8ff2d69
Keep track of the number of times we fail to populate info from a nod…
bender-the-greatest Apr 12, 2016
1a438b3
Handle flow of populate() with an exception instead of a flag
bender-the-greatest Apr 13, 2016
1e46139
Use sprintf formatting over the .format method
bender-the-greatest May 3, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions rollastic/node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class Node(dict):
Represents a cluster node.
'''

def __init__(self, cluster, node_id=None):
def __init__(self, cluster, node_id=None, bad_result_maximum=10):
'''
Init

Expand All @@ -30,6 +30,9 @@ def __init__(self, cluster, node_id=None):
'''
self.cluster = cluster
self.node_id = node_id
self.bad_result_maximum = bad_result_maximum
self.bad_result_count = 0

if self.node_id:
self.populate()

Expand All @@ -53,15 +56,26 @@ def populate(self):
'''
self.clear()

info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {})
if not info:
_LOG.warning('Bad result for node info. node_id=%s info=%s', self.node_id, info)
self.update(info)
try:
info = self.cluster.es.nodes.info(self.node_id)['nodes'].get(self.node_id, {})
self.update(info)

stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {})
if not stats:
_LOG.warning('Bad result for node stats. node_id=%s stats=%s', self.node_id, stats)
self.update(stats)
stats = self.cluster.es.nodes.stats(self.node_id)['nodes'].get(self.node_id, {})
self.update(stats)

if not stats or info:
raise Exception("Failed to retrieve values from node. node_id=%s info=%s stats=%s",
self.node_id, info, stats)

# reset the counter to zero because we only care about tracking consecutive bad results from the node
self.bad_result_count = 0
except Exception as e:
self.bad_result_count += 1
_LOG.warning("{0} (failure {1} of {2})".format(str(e), self.bad_result_count, self.bad_result_maximum))

if self.bad_result_count > self.bad_result_maximum:
raise Exception("Reached max failure count when retrieving information from node. node_id=%s",
self.node_id)

@property
def name(self):
Expand Down