Skip to content

fix(log-analysis): rectify traceback extraction logic and add root stacktrace support #628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
akshayrw25 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(log-analysis): rectify traceback extraction logic and add root stacktrace support #628

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 38 additions & 33 deletions libraries/RW/LogAnalysis/python/fetch_tracebacks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,45 +36,50 @@ def extract_traceback_from_dict_log(self, log_str: str) -> List[dict]:
# log statement is of type {"event": "some message containing data {"exception": "exception message", "stacktrace": "full stacktrace"...}", ...}
event_msg = log_str_json_obj.get("event", "")
timestamp = log_str_json_obj.get("timestamp", "")
if event_msg and all(tb_pattern in event_msg for tb_pattern in ["error handling", "with data {"]):
# the error/stacktrace content begins with "with data {...}"
traceback_dict_start_idx = event_msg.find("with data {")
traceback_dict_str_temp = event_msg[traceback_dict_start_idx+len("with data "):]
tb_dict_str_start, tb_dict_str_end = longest_balanced_curlies_sequence(traceback_dict_str_temp)
if tb_dict_str_start != -1 and tb_dict_str_end != -1:
tb_dict_str = traceback_dict_str_temp[tb_dict_str_start:tb_dict_str_end]
try:
tb_dict = json.loads(tb_dict_str)
stacktrace_str = tb_dict.get("stacktrace", "")
if stacktrace_str:
# no TIMESTAMP for now
tracebacks.append({"timestamp": timestamp, "stacktrace": stacktrace_str})
except Exception as tb_dict_parse_excp:
# report this to the exception block of tracebacks
logger.error(f"Couldn't catch the following log_str as a valid python stacktrace.\n"
f"Exception while parsing tb_dict_str from python traceback extractor: {tb_dict_parse_excp}\n"
f"\tlog_str: {tb_dict_str[:100]}...\n")

# TODO: if tracebacks are reported, runner-worker may start to have too many tracebacks leading to a traceback SLI alert.
# logger.error(''.join(traceback.format_exception(type(tb_dict_parse_excp), tb_dict_parse_excp, tb_dict_parse_excp.__traceback__)))

# check if tb_dict_str has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(tb_dict_str)

# Step 1: Extract from Event
if event_msg:
if all(tb_pattern in event_msg for tb_pattern in ["error handling", "with data {"]):
# the error/stacktrace content begins with "with data {...}"
traceback_dict_start_idx = event_msg.find("with data {")
traceback_dict_str_temp = event_msg[traceback_dict_start_idx+len("with data "):]
tb_dict_str_start, tb_dict_str_end = longest_balanced_curlies_sequence(traceback_dict_str_temp)
if tb_dict_str_start != -1 and tb_dict_str_end != -1:
tb_dict_str = traceback_dict_str_temp[tb_dict_str_start:tb_dict_str_end]
try:
tb_dict = json.loads(tb_dict_str)
stacktrace_str = tb_dict.get("stacktrace", "")
if stacktrace_str:
# no TIMESTAMP for now
tracebacks.append({"timestamp": timestamp, "stacktrace": stacktrace_str})
except Exception as tb_dict_parse_excp:
# report this to the exception block of tracebacks
logger.error(f"Couldn't catch the following log_str as a valid python stacktrace.\n"
f"Exception while parsing tb_dict_str from python traceback extractor: {tb_dict_parse_excp}\n"
f"\tlog_str: {tb_dict_str[:100]}...\n")

# check if tb_dict_str has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(tb_dict_str)
if curr_log_tracebacks:
tracebacks.extend(curr_log_tracebacks)
else:
# check if traceback_dict_str_temp has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(traceback_dict_str_temp)
if curr_log_tracebacks:
tracebacks.extend(curr_log_tracebacks)
else:
# check if traceback_dict_str_temp has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(traceback_dict_str_temp)
# check if event_msg has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(event_msg)
if curr_log_tracebacks:
tracebacks.extend(curr_log_tracebacks)
else:
# check if event_msg has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(event_msg)
if curr_log_tracebacks:
tracebacks.extend(curr_log_tracebacks)
# Step 2: Extract from Root (New Case)
root_stacktrace = log_str_json_obj.get("stacktrace", "")
if root_stacktrace:
tracebacks.append({"timestamp": timestamp, "stacktrace": root_stacktrace})
else:
# check if log_str_json_obj has traceback patterns and store'em
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(log_str_json_obj)
# check if log_str_json_obj has traceback patterns and store'em (if it's not a dict, e.g. a string)
curr_log_tracebacks = self.timestamped_log_tb_extractor.extract_from_string(str(log_str_json_obj))
if curr_log_tracebacks:
tracebacks.extend(curr_log_tracebacks)
except Exception as excp:
Expand Down