From 4d2279679165937a9d7a11302960c5d8f8c39824 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 24 Jun 2019 17:47:31 +0300 Subject: [PATCH 01/59] add "Action" and "ActionMap" classes --- cloudshell/cli/service/action_map.py | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 cloudshell/cli/service/action_map.py diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py new file mode 100644 index 0000000..e54dc9e --- /dev/null +++ b/cloudshell/cli/service/action_map.py @@ -0,0 +1,144 @@ +from collections import OrderedDict +import re + + +class Action(object): + def __init__(self, pattern, callback, execute_once=False): + """ + + :param str pattern: + :param function callback: + :param bool execute_once: + """ + self.pattern = pattern + self.callback = callback + self.execute_once = execute_once + + def __call__(self, session, logger): + """ + + :param cloudshell.cli.session.expect_session.ExpectSession session: + :param logging.Logger logger: + :return: + """ + return self.callback(session, logger) + + def __repr__(self): + """ + + :rtype: str + """ + return "{} pattern: {}, execute once: {}".format(super(Action, self).__repr__(), + self.pattern, + self.execute_once) + + def match(self, output): + """ + + :param str output: + :rtype: bool + """ + return bool(re.search(self.pattern, output, re.DOTALL)) + + +class ActionMap(object): + def __init__(self, actions=None): + """ + + :param list[Action] actions: + """ + if actions is None: + actions = [] + + self.matched_patterns = set() + self._actions_dict = OrderedDict([(action.pattern, action) for action in actions]) + + @property + def actions(self): + """ + + :rtype: list[Action] + """ + return [action for action in self._actions_dict.values()] + + @property + def active_actions(self): + """ + + :rtype: list[Action] + """ + return [action for action in self.actions if (not action.execute_once or + action.pattern not in self.matched_patterns)] + + def add(self, action): + """ + + :param Action action: + :return: + """ + self._actions_dict[action.pattern] = action + + def extend(self, action_map, override=False): + """ + + :param ActionMap action_map: + :param bool override: + :return: + """ + for action in action_map.actions: + if not override and action.pattern in self._actions_dict: + continue + self.add(action) + + self.matched_patterns |= action_map.matched_patterns + + def __call__(self, session, logger, output): + """ + + :param cloudshell.cli.session.expect_session.ExpectSession session: + :param logging.Logger logger: + :param str output: + :return: + """ + for action in self.active_actions: + if action.match(output): + self.matched_patterns.add(action.pattern) + return action(session, logger) + + def __add__(self, other): + """ + + :param other: + :rtype: ActionMap + """ + if isinstance(other, type(self)): + return ActionMap(actions=self.actions + other.actions) + + raise TypeError("unsupported operand type(s) for +: '{}' and '{}'".format(type(self), type(other))) + + def __repr__(self): + """ + + :rtype: str + """ + return "{} matched patterns: {}, actions: {}".format(super(ActionMap, self).__repr__(), + self.matched_patterns, + self.actions) + + +# if __name__ == "__main__": +# action_map1 = ActionMap(actions=[Action(pattern='action1', callback=lambda session, logger: session), +# Action(pattern='action2', callback=lambda session, logger: session)]) +# +# action_map1.add(Action(pattern="action3", +# callback=lambda session, logger: session, +# execute_once=True)) +# +# action_map2 = ActionMap(actions=[Action(pattern='action10', callback=lambda session, logger: session), +# Action(pattern='action20', callback=lambda session, logger: session)]) +# +# action_map2.add(Action(pattern="action1", +# callback=lambda session, logger: session, +# execute_once=True)) +# +# action_map2.extend(action_map1) From ff5ec07c325d11a8a1ec47c3ae0987818d5537ae Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jun 2019 16:27:24 +0300 Subject: [PATCH 02/59] update ActionMap usage in the "hardware_expect" method --- cloudshell/cli/service/action_map.py | 86 ++++++++++++++++----- cloudshell/cli/session/expect_session.py | 98 ++++-------------------- 2 files changed, 83 insertions(+), 101 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index e54dc9e..09c644d 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -1,6 +1,8 @@ from collections import OrderedDict import re +from cloudshell.cli.session.session_exceptions import SessionLoopDetectorException + class Action(object): def __init__(self, pattern, callback, execute_once=False): @@ -92,18 +94,32 @@ def extend(self, action_map, override=False): self.matched_patterns |= action_map.matched_patterns - def __call__(self, session, logger, output): + def __call__(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ :param cloudshell.cli.session.expect_session.ExpectSession session: :param logging.Logger logger: :param str output: - :return: + :param bool check_action_loop_detector: + :param ActionLoopDetector action_loop_detector: + :rtype: bool """ for action in self.active_actions: if action.match(output): + logger.debug("Matched Action with pattern: {}".format(action.pattern)) + + if check_action_loop_detector: + logger.debug("Checking loops fro Action with pattern : {}".format(action.pattern)) + + if action_loop_detector.loops_detected(action.pattern): + logger.error("Loops detected for action patter: {}".format(action.pattern)) + raise SessionLoopDetectorException("Expected actions loops detected") + + action(session, logger) self.matched_patterns.add(action.pattern) - return action(session, logger) + return True + + return False def __add__(self, other): """ @@ -126,19 +142,51 @@ def __repr__(self): self.actions) -# if __name__ == "__main__": -# action_map1 = ActionMap(actions=[Action(pattern='action1', callback=lambda session, logger: session), -# Action(pattern='action2', callback=lambda session, logger: session)]) -# -# action_map1.add(Action(pattern="action3", -# callback=lambda session, logger: session, -# execute_once=True)) -# -# action_map2 = ActionMap(actions=[Action(pattern='action10', callback=lambda session, logger: session), -# Action(pattern='action20', callback=lambda session, logger: session)]) -# -# action_map2.add(Action(pattern="action1", -# callback=lambda session, logger: session, -# execute_once=True)) -# -# action_map2.extend(action_map1) +class ActionLoopDetector(object): + """Help to detect loops for action combinations""" + + def __init__(self, max_loops, max_combination_length): + """ + + :param max_loops: + :param max_combination_length: + :return: + """ + self._max_action_loops = max_loops + self._max_combination_length = max_combination_length + self._action_history = [] + + def loops_detected(self, action_pattern): + """Add action key to the history and detect loops + + :param str action_pattern: + :return: + """ + self._action_history.append(action_pattern) + for combination_length in range(1, self._max_combination_length + 1): + if self._is_combination_compatible(combination_length): + if self._is_loop_exists(combination_length): + return True + return False + + def _is_combination_compatible(self, combination_length): + """Check if combinations may exist + + :param combination_length: + :return: + """ + return len(self._action_history) / combination_length >= self._max_action_loops + + def _is_loop_exists(self, combination_length): + """Detect loops for combination length + + :param combination_length: + :return: + """ + reversed_history = self._action_history[::-1] + combinations = [reversed_history[x:x + combination_length] for x in + range(0, len(reversed_history), combination_length)][:self._max_action_loops] + for x, y in [combinations[x:x + 2] for x in range(0, len(combinations) - 1)]: + if x != y: + return False + return True diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 8c917d7..c252db4 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,11 +1,12 @@ import re import time from abc import ABCMeta, abstractmethod -from collections import OrderedDict +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.action_map import ActionLoopDetector from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer from cloudshell.cli.session.session import Session -from cloudshell.cli.session.session_exceptions import SessionLoopDetectorException, SessionLoopLimitException, \ +from cloudshell.cli.session.session_exceptions import SessionLoopLimitException, \ ExpectedSessionException, CommandExecutionException, SessionReadTimeout, SessionReadEmptyData @@ -181,7 +182,6 @@ def match_prompt(self, prompt, match_string, logger): def hardware_expect(self, command, expected_string, logger, action_map=None, error_map=None, timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, remove_command_from_output=True, **optional_args): - """Get response form the device and compare it to action_map, error_map and expected_string patterns, perform actions specified in action_map if any, and return output. Raise Exception if receive empty response from device within a minute @@ -189,7 +189,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err :param command: command to send :param expected_string: expected string :param logger: logger - :param action_map: dict with {re_str: action} to trigger some action on received string + :param action_map: ActionMap :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, CommandExecutionException|str] :param timeout: session timeout @@ -201,10 +201,10 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err """ if not action_map: - action_map = OrderedDict() + action_map = ActionMap() if not error_map: - error_map = OrderedDict() + error_map = ActionMap() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout @@ -227,6 +227,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err action_loop_detector = ActionLoopDetector(self._loop_detector_max_action_loops, self._loop_detector_max_combination_length) + while retries == 0 or retries_count < retries: # try: @@ -258,20 +259,15 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err output_list.append(output_str) is_correct_exit = True - for action_key in action_map: - result_match = re.search(action_key, output_str, re.DOTALL) - if result_match: - output_list.append(output_str) - - if check_action_loop_detector: - if action_loop_detector.loops_detected(action_key): - logger.error('Loops detected') - raise SessionLoopDetectorException(self.__class__.__name__, - 'Expected actions loops detected') - logger.debug('Action key: {}'.format(action_key)) - action_map[action_key](self, logger) - output_str = '' - break + action_matched = action_map(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=check_action_loop_detector, + action_loop_detector=action_loop_detector) + + if action_matched: + output_list.append(output_str) + output_str = '' if is_correct_exit: break @@ -316,65 +312,3 @@ def reconnect(self, prompt, logger, timeout=None): logger.debug(e) raise ExpectedSessionException(self.__class__.__name__, 'Reconnect unsuccessful, timeout exceeded, see logs for more details') - - -class ActionLoopDetector(object): - """Help to detect loops for action combinations""" - - def __init__(self, max_loops, max_combination_length): - """ - - :param max_loops: - :param max_combination_length: - :return: - """ - self._max_action_loops = max_loops - self._max_combination_length = max_combination_length - self._action_history = [] - - def loops_detected(self, action_key): - """ - Add action key to the history and detect loops - - :param action_key: - :return: - """ - # """Added action key to the history and detect for loops""" - loops_detected = False - self._action_history.append(action_key) - for combination_length in range(1, self._max_combination_length + 1): - if self._is_combination_compatible(combination_length): - if self._detect_loops_for_combination_length(combination_length): - loops_detected = True - break - return loops_detected - - def _is_combination_compatible(self, combination_length): - """ - Check if combinations may exist - - :param combination_length: - :return: - """ - if len(self._action_history) / combination_length >= self._max_action_loops: - is_compatible = True - else: - is_compatible = False - return is_compatible - - def _detect_loops_for_combination_length(self, combination_length): - """ - Detect loops for combination length - - :param combination_length: - :return: - """ - reversed_history = self._action_history[::-1] - combinations = [reversed_history[x:x + combination_length] for x in - range(0, len(reversed_history), combination_length)][:self._max_action_loops] - is_loops_exist = True - for x, y in [combinations[x:x + 2] for x in range(0, len(combinations) - 1)]: - if x != y: - is_loops_exist = False - break - return is_loops_exist From 4871fee89f9ccd29a0f985e445ee122a7575fafa Mon Sep 17 00:00:00 2001 From: anthony Date: Wed, 26 Jun 2019 23:23:41 +0300 Subject: [PATCH 03/59] replace Action map OrderedDict with ActionMap class --- .../cli/command_template/command_template.py | 43 +++++++++------ .../command_template_executor.py | 53 ++++++------------- cloudshell/cli/service/cli_service_impl.py | 4 +- cloudshell/cli/service/command_mode.py | 43 +++++++-------- cloudshell/cli/session/expect_session.py | 21 ++++---- cloudshell/cli/session/telnet_session.py | 23 +++++--- 6 files changed, 92 insertions(+), 95 deletions(-) diff --git a/cloudshell/cli/command_template/command_template.py b/cloudshell/cli/command_template/command_template.py index 90020e3..8ce3416 100644 --- a/cloudshell/cli/command_template/command_template.py +++ b/cloudshell/cli/command_template/command_template.py @@ -1,44 +1,52 @@ from collections import OrderedDict import re +from cloudshell.cli.service.action_map import ActionMap + class CommandTemplate: def __init__(self, command, action_map=None, error_map=None): - """Command Template. + """ - :type command: str - :type action_map: dict + :param str command: + :param cloudshell.cli.service.action_map.ActionMap action_map: :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] """ self._command = command - self._action_map = action_map or OrderedDict() + self._action_map = action_map or ActionMap() self._error_map = error_map or OrderedDict() @property def action_map(self): """ - Property for action map - :return: - :rtype: OrderedDict() + + :rtype: cloudshell.cli.service.action_map.ActionMap """ return self._action_map @property def error_map(self): """ - Property for error map - :return: - :rtype: OrderedDict + + :rtype: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] """ return self._error_map # ToDo: Needs to be reviewed def get_command(self, **kwargs): - action_map = (OrderedDict(kwargs.get('action_map', None) or OrderedDict())) - action_map.update(self._action_map) - error_map = OrderedDict(self._error_map) - error_map.update(kwargs.get('error_map', None) or OrderedDict()) + """ + + :param dict kwargs: + :rtype: dict + """ + # todo: verify action map creation + action_map = kwargs.get('action_map') or ActionMap() + action_map.extend(self.action_map) + + error_map = kwargs.get("error_map") or OrderedDict() + error_map.update(self.error_map) + return { 'command': self.prepare_command(**kwargs), 'action_map': action_map, @@ -46,6 +54,11 @@ def get_command(self, **kwargs): } def prepare_command(self, **kwargs): + """ + + :param dict kwargs: + :rtype: str + """ cmd = self._command keys = re.findall(r"{(\w+)}", self._command) for key in keys: @@ -53,7 +66,7 @@ def prepare_command(self, **kwargs): cmd = re.sub(r"\[[^[]*?{{{key}}}.*?\]".format(key=key), r"", cmd) if not cmd: - raise Exception(self.__class__.__name__, 'Unable to prepare command') + raise Exception("Unable to prepare command") cmd = re.sub(r"\s+", " ", cmd).strip(' \t\n\r') result = re.sub(r"\[|\]", "", cmd).format(**kwargs) diff --git a/cloudshell/cli/command_template/command_template_executor.py b/cloudshell/cli/command_template/command_template_executor.py index a0e34e8..575c3f1 100644 --- a/cloudshell/cli/command_template/command_template_executor.py +++ b/cloudshell/cli/command_template/command_template_executor.py @@ -1,60 +1,41 @@ from collections import OrderedDict -from cloudshell.cli.command_template.command_template import CommandTemplate +from cloudshell.cli.service.action_map import ActionMap class CommandTemplateExecutor(object): - """ - Execute command template using cli service - """ + """Execute command template using cli service""" def __init__(self, cli_service, command_template, action_map=None, error_map=None, **optional_kwargs): """ - :param cli_service: - :type cli_service: CliService - :param command_template: - :type command_template: CommandTemplate + + :param cloudshell.cli.service.cli_service.CliService cli_service: + :param cloudshell.cli.command_template.command_template.CommandTemplate command_template: + :param cloudshell.cli.service.action_map.ActionMap action_map: :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] :return: """ self._cli_service = cli_service self._command_template = command_template - self._action_map = action_map or OrderedDict() - self._error_map = error_map or OrderedDict() - self._optional_kwargs = optional_kwargs - @property - def action_map(self): - """ - Return updated action - """ - return dict(**self._action_map, **self._command_template.action_map) + self._action_map = action_map or ActionMap() + self._action_map.extend(command_template.action_map) - @property - def error_map(self): - return dict(**self._error_map, **self._command_template.error_map) + self._error_map = error_map or OrderedDict() + self._error_map.update(command_template.error_map) - @property - def optional_kwargs(self): - return self._optional_kwargs + self._optional_kwargs = optional_kwargs def execute_command(self, **command_kwargs): """ - Execute command - :param command_kwargs: + + :param dict command_kwargs: :return: Command output :rtype: str """ command = self._command_template.prepare_command(**command_kwargs) - return self._cli_service.send_command(command, action_map=self.action_map, error_map=self.error_map, - **self.optional_kwargs) - - def update_action_map(self, action_map): - self._action_map.update(action_map) - - def update_error_map(self, error_map): - self._error_map.update(error_map) - - def update_optional_kwargs(self, **optional_kwargs): - self.optional_kwargs.update(optional_kwargs) + return self._cli_service.send_command(command, + action_map=self._action_map, + error_map=self._error_map, + **self._optional_kwargs) diff --git a/cloudshell/cli/service/cli_service_impl.py b/cloudshell/cli/service/cli_service_impl.py index 582c0c6..ee11c58 100644 --- a/cloudshell/cli/service/cli_service_impl.py +++ b/cloudshell/cli/service/cli_service_impl.py @@ -104,10 +104,10 @@ def enter_mode(self, command_mode): def send_command(self, command, expected_string=None, action_map=None, error_map=None, logger=None, remove_prompt=False, *args, **kwargs): """ - Send command + :param command: :param expected_string: - :param action_map: + :param cloudshell.cli.service.action_map.ActionMap action_map: :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] :param logger: diff --git a/cloudshell/cli/service/command_mode.py b/cloudshell/cli/service/command_mode.py index 67b6f2d..defd4f1 100755 --- a/cloudshell/cli/service/command_mode.py +++ b/cloudshell/cli/service/command_mode.py @@ -1,5 +1,6 @@ import re +from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.service.cli_exception import CliException from cloudshell.cli.service.node import Node @@ -9,9 +10,7 @@ class CommandModeException(CliException): class CommandMode(Node): - """ - Class describes our prompt and implement enter and exit command functions - """ + """Class describes our prompt and implement enter and exit command functions""" RELATIONS_DICT = {} @@ -19,34 +18,30 @@ def __init__(self, prompt, enter_command=None, exit_command=None, enter_action_m enter_error_map=None, exit_error_map=None, parent_mode=None, enter_actions=None, use_exact_prompt=False): """ - :param prompt: Prompt of this mode - :type prompt: str - :param enter_command: Command used to enter this mode - :type enter_command: str - :param exit_command: Command used to exit from this mode - :type exit_command: str - :param enter_actions: Actions which needs to be done when entering this mode - :param enter_action_map: Enter expected actions - :type enter_action_map: dict - :param enter_error_map: expected error map with subclass of CommandExecutionException or str - :type enter_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] - :param exit_action_map: - :type exit_action_map: dict - :param exit_error_map: expected error map with subclass of CommandExecutionException or str - :type exit_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] - :param - :param parent_mode: Connect parent mode - """ + + :param str prompt: Prompt of this mode + :param str enter_command: Command used to enter this mode + :param str exit_command: Command used to exit from this mode + :param enter_actions: Actions which needs to be done when entering this mode + :param cloudshell.cli.service.action_map.ActionMap enter_action_map: Enter expected actions + :param enter_error_map: expected error map with subclass of CommandExecutionException or str + :type enter_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.action_map.ActionMap exit_action_map: + :param exit_error_map: expected error map with subclass of CommandExecutionException or str + :type exit_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param parent_mode: Connect parent mode + """ + super(CommandMode, self).__init__() + if not exit_error_map: exit_error_map = {} if not enter_error_map: enter_error_map = {} if not exit_action_map: - exit_action_map = {} + exit_action_map = ActionMap() if not enter_action_map: - enter_action_map = {} + enter_action_map = ActionMap() - super(CommandMode, self).__init__() self._prompt = prompt self._exact_prompt = None self._enter_command = enter_command diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index c252db4..0cc596c 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -183,23 +183,22 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, remove_command_from_output=True, **optional_args): """Get response form the device and compare it to action_map, error_map and expected_string patterns, + perform actions specified in action_map if any, and return output. Raise Exception if receive empty response from device within a minute - - :param command: command to send - :param expected_string: expected string - :param logger: logger - :param action_map: ActionMap + :param str command: command to send + :param str expected_string: expected string + :param logging.Logger logger: logger + :param cloudshell.cli.service.action_map.ActionMap action_map: :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, CommandExecutionException|str] - :param timeout: session timeout - :param retries: maximal retries count - :param remove_command_from_output: In some switches the output string includes the command which was called. - The flag used to verify whether the the command string removed from the output string. - :return: + :param int timeout: session timeout + :param int retries: maximal retries count + :param bool check_action_loop_detector: + :param bool remove_command_from_output: In some switches the output string includes the command which was + called. The flag used to verify whether the the command string removed from the output string. :rtype: str """ - if not action_map: action_map = ActionMap() diff --git a/cloudshell/cli/session/telnet_session.py b/cloudshell/cli/session/telnet_session.py index 840ee66..73ae7fe 100644 --- a/cloudshell/cli/session/telnet_session.py +++ b/cloudshell/cli/session/telnet_session.py @@ -1,7 +1,8 @@ import socket import telnetlib -from collections import OrderedDict +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.session.connection_params import ConnectionParams from cloudshell.cli.session.expect_session import ExpectSession from cloudshell.cli.session.session_exceptions import SessionException, SessionReadTimeout, SessionReadEmptyData @@ -41,12 +42,20 @@ def __del__(self): self.disconnect() def _connect_actions(self, prompt, logger): - action_map = OrderedDict() - action_map['[Ll]ogin:|[Uu]ser:|[Uu]sername:'] = lambda session, logger: session.send_line(session.username, - logger) - action_map['[Pp]assword:'] = lambda session, logger: session.send_line(session.password, logger) - self.hardware_expect(None, expected_string=prompt, timeout=self._timeout, logger=logger, - action_map=action_map) + """ + + :param str prompt: + :param logging.Logger logger: + :return: + """ + action_map = ActionMap(actions=[Action(pattern="[Ll]ogin:|[Uu]ser:|[Uu]sername:", + callback=lambda session, logger: + session.send_line(session.username, logger)), + Action(pattern="[Pp]assword:", + callback=lambda session, logger: + session.send_line(session.password, logger))]) + + self.hardware_expect(None, expected_string=prompt, timeout=self._timeout, logger=logger, action_map=action_map) self._on_session_start(logger) def _initialize_session(self, prompt, logger): From 85b9efac00d17019d5ea3e2e01e7908350616f99 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 12:45:40 +0300 Subject: [PATCH 04/59] fix error map initialization --- cloudshell/cli/session/expect_session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 0cc596c..10ba2f6 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,6 +1,7 @@ import re import time from abc import ABCMeta, abstractmethod +from collections import OrderedDict from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.service.action_map import ActionLoopDetector @@ -203,7 +204,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err action_map = ActionMap() if not error_map: - error_map = ActionMap() + error_map = OrderedDict() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout From 4324f8f4a27ad7f5b3562c7ec970987e4e9bdc9e Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 12:58:42 +0300 Subject: [PATCH 05/59] replace "super(Class, self)" calls with "super()" call --- cloudshell/cli/service/action_map.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 09c644d..2f39624 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -30,9 +30,7 @@ def __repr__(self): :rtype: str """ - return "{} pattern: {}, execute once: {}".format(super(Action, self).__repr__(), - self.pattern, - self.execute_once) + return "{} pattern: {}, execute once: {}".format(super().__repr__(), self.pattern, self.execute_once) def match(self, output): """ @@ -137,9 +135,7 @@ def __repr__(self): :rtype: str """ - return "{} matched patterns: {}, actions: {}".format(super(ActionMap, self).__repr__(), - self.matched_patterns, - self.actions) + return "{} matched patterns: {}, actions: {}".format(super().__repr__(), self.matched_patterns, self.actions) class ActionLoopDetector(object): From 695612bdc90e7efc563e2c6c9ccfe478c6dc185f Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 13:23:11 +0300 Subject: [PATCH 06/59] use f-Strings instead of str.format --- cloudshell/cli/service/action_map.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 2f39624..fd98a92 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -30,7 +30,7 @@ def __repr__(self): :rtype: str """ - return "{} pattern: {}, execute once: {}".format(super().__repr__(), self.pattern, self.execute_once) + return f"{super().__repr__()} pattern: {self.pattern}, execute once: {self.execute_once}" def match(self, output): """ @@ -104,13 +104,13 @@ def __call__(self, session, logger, output, check_action_loop_detector, action_l """ for action in self.active_actions: if action.match(output): - logger.debug("Matched Action with pattern: {}".format(action.pattern)) + logger.debug(f"Matched Action with pattern: {action.pattern}") if check_action_loop_detector: - logger.debug("Checking loops fro Action with pattern : {}".format(action.pattern)) + logger.debug(f"Checking loops for Action with pattern : {action.pattern}") if action_loop_detector.loops_detected(action.pattern): - logger.error("Loops detected for action patter: {}".format(action.pattern)) + logger.error(f"Loops detected for action patter: {action.pattern}") raise SessionLoopDetectorException("Expected actions loops detected") action(session, logger) @@ -128,14 +128,14 @@ def __add__(self, other): if isinstance(other, type(self)): return ActionMap(actions=self.actions + other.actions) - raise TypeError("unsupported operand type(s) for +: '{}' and '{}'".format(type(self), type(other))) + raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") def __repr__(self): """ :rtype: str """ - return "{} matched patterns: {}, actions: {}".format(super().__repr__(), self.matched_patterns, self.actions) + return f"{super().__repr__()} matched patterns: {self.matched_patterns}, actions: {self.actions}" class ActionLoopDetector(object): From 7fd66e9201c33954afd5412776e2606d27daead8 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 16:39:41 +0300 Subject: [PATCH 07/59] add unit tests for the "action_map" module --- tests/cli/service/__init__.py | 0 tests/cli/service/test_action_map.py | 132 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/cli/service/__init__.py create mode 100644 tests/cli/service/test_action_map.py diff --git a/tests/cli/service/__init__.py b/tests/cli/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/service/test_action_map.py b/tests/cli/service/test_action_map.py new file mode 100644 index 0000000..ad72e56 --- /dev/null +++ b/tests/cli/service/test_action_map.py @@ -0,0 +1,132 @@ +import unittest +from unittest import mock + +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap + + +class TestAction(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.callback = mock.MagicMock() + self.action = Action(pattern="test pattern", callback=self.callback) + + def test_call(self): + """Check that method will call callback function with session and logger as attributes""" + # act + self.action(session=self.session, logger=self.logger) + # verify + self.callback.assert_called_once_with(self.session, self.logger) + + @mock.patch("cloudshell.cli.service.action_map.re") + def test_match_return_true(self, re): + """Check that method will return True if output matches pattern""" + re.search.return_value = True + output = "test output" + # act + result = self.action.match(output) + # verify + self.assertTrue(result) + + @mock.patch("cloudshell.cli.service.action_map.re") + def test_match_return_false(self, re): + """Check that method will return False if output doesn't match pattern""" + re.search.return_value = False + output = "test output" + # act + result = self.action.match(output) + # verify + self.assertFalse(result) + + +class TestActionMap(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.callback = mock.MagicMock() + self.actions = [Action(pattern="[Pp]attern 1", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback, execute_once=True), + Action(pattern="[Pp]attern 3", callback=self.callback, execute_once=True)] + + self.action_map = ActionMap(actions=self.actions) + + def test_actions(self): + """Check that method will return all actions""" + action1, action2, action3 = self.actions + self.action_map.matched_patterns = {action1.pattern, action2.pattern} + # verify + self.assertEqual(self.action_map.actions, [action1, action2, action3]) + + def test_active_actions(self): + """Check that method will return only active actions""" + action1, action2, action3 = self.actions + self.action_map.matched_patterns = {action1.pattern, action2.pattern} + # verify + self.assertEqual(self.action_map.active_actions, [action1, action3]) + + def test_extend(self): + """Check that extend will add new actions and will not override existing one""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + self.action_map.extend(action_map=default_action_map) + + # verify + self.assertEqual(self.action_map.matched_patterns, {default_action1.pattern, + default_action2.pattern, + action3.pattern}) + + self.assertEqual(self.action_map.actions, [action1, action2, action3, default_action1]) + + def test_extend_with_override_true(self): + """Check that extend will add new actions and will not override existing one""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + self.action_map.extend(action_map=default_action_map, override=True) + + # verify + self.assertEqual(self.action_map.matched_patterns, {default_action1.pattern, + default_action2.pattern, + action3.pattern}) + + self.assertEqual(self.action_map.actions, [action1, default_action2, action3, default_action1]) + + def test_add(self): + """Check that __add__ method will create new ActionMap""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + result = self.action_map + default_action_map + + # verify + self.assertEqual(result.matched_patterns, set()) + self.assertEqual(result.actions, [action1, action2, action3, default_action1]) From 0deac1bfa2d1953d2a235e5f797b6ce584d6cdfa Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 16:59:04 +0300 Subject: [PATCH 08/59] update ActionMap.__add__ method logic --- cloudshell/cli/service/action_map.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index fd98a92..8391447 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -126,7 +126,10 @@ def __add__(self, other): :rtype: ActionMap """ if isinstance(other, type(self)): - return ActionMap(actions=self.actions + other.actions) + actions = self.actions + [action for action in other.actions if action.pattern not in + [action.pattern for action in self.actions]] + + return ActionMap(actions=actions) raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") From 97984d0726c5e78184b772e0f75dd9493859fa03 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 15:50:58 +0300 Subject: [PATCH 09/59] add "Error" and "ErrorMap" classes --- .../cli/command_template/command_template.py | 13 +- .../command_template_executor.py | 10 +- cloudshell/cli/service/cli_service_impl.py | 3 +- cloudshell/cli/service/command_mode.py | 11 +- cloudshell/cli/service/error_map.py | 118 ++++++++++++++++++ cloudshell/cli/session/expect_session.py | 4 +- 6 files changed, 136 insertions(+), 23 deletions(-) create mode 100644 cloudshell/cli/service/error_map.py diff --git a/cloudshell/cli/command_template/command_template.py b/cloudshell/cli/command_template/command_template.py index 8ce3416..f2a601d 100644 --- a/cloudshell/cli/command_template/command_template.py +++ b/cloudshell/cli/command_template/command_template.py @@ -2,6 +2,7 @@ import re from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap class CommandTemplate: @@ -10,12 +11,11 @@ def __init__(self, command, action_map=None, error_map=None): :param str command: :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: """ self._command = command self._action_map = action_map or ActionMap() - self._error_map = error_map or OrderedDict() + self._error_map = error_map or ErrorMap() @property def action_map(self): @@ -29,7 +29,7 @@ def action_map(self): def error_map(self): """ - :rtype: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :rtype: cloudshell.cli.service.error_map.ErrorMap """ return self._error_map @@ -40,12 +40,11 @@ def get_command(self, **kwargs): :param dict kwargs: :rtype: dict """ - # todo: verify action map creation action_map = kwargs.get('action_map') or ActionMap() action_map.extend(self.action_map) - error_map = kwargs.get("error_map") or OrderedDict() - error_map.update(self.error_map) + error_map = kwargs.get("error_map") or ErrorMap() + error_map.extend(self.error_map) return { 'command': self.prepare_command(**kwargs), diff --git a/cloudshell/cli/command_template/command_template_executor.py b/cloudshell/cli/command_template/command_template_executor.py index 575c3f1..ba90038 100644 --- a/cloudshell/cli/command_template/command_template_executor.py +++ b/cloudshell/cli/command_template/command_template_executor.py @@ -1,6 +1,5 @@ -from collections import OrderedDict - from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap class CommandTemplateExecutor(object): @@ -12,8 +11,7 @@ def __init__(self, cli_service, command_template, action_map=None, error_map=Non :param cloudshell.cli.service.cli_service.CliService cli_service: :param cloudshell.cli.command_template.command_template.CommandTemplate command_template: :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: :return: """ self._cli_service = cli_service @@ -22,8 +20,8 @@ def __init__(self, cli_service, command_template, action_map=None, error_map=Non self._action_map = action_map or ActionMap() self._action_map.extend(command_template.action_map) - self._error_map = error_map or OrderedDict() - self._error_map.update(command_template.error_map) + self._error_map = error_map or ErrorMap() + self._error_map.extend(command_template.error_map) self._optional_kwargs = optional_kwargs diff --git a/cloudshell/cli/service/cli_service_impl.py b/cloudshell/cli/service/cli_service_impl.py index ee11c58..0726bc0 100644 --- a/cloudshell/cli/service/cli_service_impl.py +++ b/cloudshell/cli/service/cli_service_impl.py @@ -108,8 +108,7 @@ def send_command(self, command, expected_string=None, action_map=None, error_map :param command: :param expected_string: :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: :param logger: :param remove_prompt: :param args: diff --git a/cloudshell/cli/service/command_mode.py b/cloudshell/cli/service/command_mode.py index defd4f1..2fcd0e9 100755 --- a/cloudshell/cli/service/command_mode.py +++ b/cloudshell/cli/service/command_mode.py @@ -1,6 +1,7 @@ import re from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.service.cli_exception import CliException from cloudshell.cli.service.node import Node @@ -24,19 +25,17 @@ def __init__(self, prompt, enter_command=None, exit_command=None, enter_action_m :param str exit_command: Command used to exit from this mode :param enter_actions: Actions which needs to be done when entering this mode :param cloudshell.cli.service.action_map.ActionMap enter_action_map: Enter expected actions - :param enter_error_map: expected error map with subclass of CommandExecutionException or str - :type enter_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap enter_error_map: :param cloudshell.cli.service.action_map.ActionMap exit_action_map: - :param exit_error_map: expected error map with subclass of CommandExecutionException or str - :type exit_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap exit_error_map: :param parent_mode: Connect parent mode """ super(CommandMode, self).__init__() if not exit_error_map: - exit_error_map = {} + exit_error_map = ErrorMap() if not enter_error_map: - enter_error_map = {} + enter_error_map = ErrorMap() if not exit_action_map: exit_action_map = ActionMap() if not enter_action_map: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py new file mode 100644 index 0000000..2b6b4e4 --- /dev/null +++ b/cloudshell/cli/service/error_map.py @@ -0,0 +1,118 @@ +from collections import OrderedDict +import re + +from cloudshell.cli.session.session_exceptions import CommandExecutionException + + +class Error: + def __init__(self, pattern, error): + """ + + :param str pattern: + :param str error: + """ + self.pattern = pattern + self.error = error + + def __call__(self): + """ + + :raises: CommandExecutionException + """ + if isinstance(self.error, CommandExecutionException): + raise self.error + + raise CommandExecutionException(f"Session returned '{self.error}'") + + def __repr__(self): + """ + + :rtype: str + """ + return f"{super().__repr__()} pattern: {self.pattern}, error: {self.error}" + + def match(self, output): + """ + + :param str output: + :rtype: bool + """ + return bool(re.search(self.pattern, output, re.DOTALL)) + + +class ErrorMap: + def __init__(self, errors=None): + """ + + :param list[Error] errors: + """ + if errors is None: + errors = [] + + self._errors_dict = OrderedDict([(error.pattern, error) for error in errors]) + + @property + def errors(self): + """ + + :rtype: list[Error] + """ + return list(self._errors_dict.values()) + + def add(self, error): + """ + + :param Error error: + :return: + """ + self._errors_dict[error.pattern] = error + + def extend(self, error_map, override=False): + """ + + :param ErrorMap error_map: + :param bool override: + :return: + """ + for error in error_map.errors: + if not override and error.pattern in self._errors_dict: + continue + self.add(error) + + def __call__(self, output, logger): + """ + + :param str output: + :param logging.Logger logger: + :rtype: bool + """ + + for error in self.errors: + if error.match(output): + logger.debug(f"Matched Error with pattern: {error.pattern}") + + if isinstance(error, CommandExecutionException): + raise error + else: + raise CommandExecutionException('Session returned \'{}\''.format(error)) + + def __add__(self, other): + """ + + :param other: + :rtype: ActionMap + """ + if isinstance(other, type(self)): + errors = self.errors + [error for error in other.errors if error.pattern not in + [error.pattern for error in self.errors]] + + return ErrorMap(errors=errors) + + raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + + def __repr__(self): + """ + + :rtype: str + """ + return f"{super().__repr__()} errors: {self.errors}" diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 10ba2f6..5e51abf 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,9 +1,9 @@ import re import time from abc import ABCMeta, abstractmethod -from collections import OrderedDict from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.service.action_map import ActionLoopDetector from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer from cloudshell.cli.session.session import Session @@ -204,7 +204,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err action_map = ActionMap() if not error_map: - error_map = OrderedDict() + error_map = ErrorMap() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout From 29a231e78adbfd3ca269c686d36f3bd5580bff69 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 15:52:53 +0300 Subject: [PATCH 10/59] small fixes for the action_map module --- cloudshell/cli/service/action_map.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 8391447..f210f40 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -4,7 +4,7 @@ from cloudshell.cli.session.session_exceptions import SessionLoopDetectorException -class Action(object): +class Action: def __init__(self, pattern, callback, execute_once=False): """ @@ -41,7 +41,7 @@ def match(self, output): return bool(re.search(self.pattern, output, re.DOTALL)) -class ActionMap(object): +class ActionMap: def __init__(self, actions=None): """ @@ -59,7 +59,7 @@ def actions(self): :rtype: list[Action] """ - return [action for action in self._actions_dict.values()] + return list(self._actions_dict.values()) @property def active_actions(self): @@ -141,7 +141,7 @@ def __repr__(self): return f"{super().__repr__()} matched patterns: {self.matched_patterns}, actions: {self.actions}" -class ActionLoopDetector(object): +class ActionLoopDetector: """Help to detect loops for action combinations""" def __init__(self, max_loops, max_combination_length): From 58c2dcaee57f2e28d5113971162e137d6253ac63 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 16:19:51 +0300 Subject: [PATCH 11/59] update ErrorMap usage in the "hardware_expect" method --- cloudshell/cli/session/expect_session.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 5e51abf..510c1a6 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -278,14 +278,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err result_output = ''.join(output_list) - for error_pattern, error in error_map.items(): - result_match = re.search(error_pattern, result_output, re.DOTALL) - - if result_match: - if isinstance(error, CommandExecutionException): - raise error - else: - raise CommandExecutionException('Session returned \'{}\''.format(error)) + error_map(output=result_output, logger=logger) # Read buffer to the end. Useful when expected_string isn't last in buffer result_output += self._clear_buffer(self._clear_buffer_timeout, logger) From 3f170f0f9fd39f1169f7413fbe516a50ea4baad1 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 16:42:41 +0300 Subject: [PATCH 12/59] fix unittest and bugs for the "error_map" module --- cloudshell/cli/command_template/command_template.py | 1 - cloudshell/cli/service/command_mode_helper.py | 1 - cloudshell/cli/service/error_map.py | 8 ++------ cloudshell/cli/session/expect_session.py | 3 +-- tests/cli/session/test_expect_session.py | 13 +++++++++---- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/cloudshell/cli/command_template/command_template.py b/cloudshell/cli/command_template/command_template.py index f2a601d..eaf8e1d 100644 --- a/cloudshell/cli/command_template/command_template.py +++ b/cloudshell/cli/command_template/command_template.py @@ -1,4 +1,3 @@ -from collections import OrderedDict import re from cloudshell.cli.service.action_map import ActionMap diff --git a/cloudshell/cli/service/command_mode_helper.py b/cloudshell/cli/service/command_mode_helper.py index cd119bf..860d370 100644 --- a/cloudshell/cli/service/command_mode_helper.py +++ b/cloudshell/cli/service/command_mode_helper.py @@ -1,6 +1,5 @@ from collections import OrderedDict -import re from cloudshell.cli.service.command_mode import CommandMode, CommandModeException from cloudshell.cli.service.node import NodeOperations from cloudshell.cli.session.session import Session diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 2b6b4e4..16e2fe3 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -9,7 +9,7 @@ def __init__(self, pattern, error): """ :param str pattern: - :param str error: + :param str|CommandExecutionException error: """ self.pattern = pattern self.error = error @@ -90,11 +90,7 @@ def __call__(self, output, logger): for error in self.errors: if error.match(output): logger.debug(f"Matched Error with pattern: {error.pattern}") - - if isinstance(error, CommandExecutionException): - raise error - else: - raise CommandExecutionException('Session returned \'{}\''.format(error)) + error() def __add__(self, other): """ diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 510c1a6..b0124e6 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -191,8 +191,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err :param str expected_string: expected string :param logging.Logger logger: logger :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: :param int timeout: session timeout :param int retries: maximal retries count :param bool check_action_loop_detector: diff --git a/tests/cli/session/test_expect_session.py b/tests/cli/session/test_expect_session.py index d374cc6..4d0bd7e 100644 --- a/tests/cli/session/test_expect_session.py +++ b/tests/cli/session/test_expect_session.py @@ -2,6 +2,10 @@ from unittest import TestCase from unittest.mock import patch, Mock, call, MagicMock +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import Error +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.session.expect_session import ExpectSession, ActionLoopDetector from cloudshell.cli.session.session_exceptions import SessionReadTimeout, ExpectedSessionException, \ SessionLoopLimitException, CommandExecutionException @@ -215,8 +219,8 @@ def test_hardware_expect_action_map_call(self, clear_buffer, receive_all, send_l receive_all.side_effect = side_effect normalize_buffer.side_effect = side_effect test_func = Mock() - action_map = OrderedDict({fake_out: test_func}) - result = self._instance.hardware_expect(command, expected_string, self._logger, action_map=action_map) + action_map = ActionMap(actions=[Action(pattern=fake_out, callback=test_func)]) + self._instance.hardware_expect(command, expected_string, self._logger, action_map=action_map) test_func.assert_called_once_with(self._instance, self._logger) @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") @@ -228,7 +232,7 @@ def test_hardware_expect_error_map_call(self, clear_buffer, receive_all, send_li expected_string = 'test_string' receive_all.return_value = expected_string normalize_buffer.return_value = expected_string - error_map = OrderedDict({expected_string: 'test_error'}) + error_map = ErrorMap(errors=[Error(pattern=expected_string, error='test_error')]) exception = CommandExecutionException with self.assertRaises(exception): result = self._instance.hardware_expect(command, expected_string, self._logger, error_map=error_map) @@ -246,7 +250,8 @@ class TestException(CommandExecutionException): expected_string = 'test_string' receive_all.return_value = expected_string normalize_buffer.return_value = expected_string - error_map = OrderedDict({expected_string: TestException('test_error')}) + error_map = ErrorMap(errors=[Error(pattern=expected_string, error=TestException('test_error'))]) + with self.assertRaises(TestException): self._instance.hardware_expect( command, expected_string, self._logger, error_map=error_map) From 6b63e34840b192772e56da01fec142e68e5d51f2 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 17:17:37 +0300 Subject: [PATCH 13/59] add unit tests for the "error_map" module --- tests/cli/service/test_error_map.py | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/cli/service/test_error_map.py diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py new file mode 100644 index 0000000..aa9b89e --- /dev/null +++ b/tests/cli/service/test_error_map.py @@ -0,0 +1,87 @@ +import unittest +from unittest import mock + +from cloudshell.cli.service.error_map import Error +from cloudshell.cli.service.error_map import ErrorMap +from cloudshell.cli.session.session_exceptions import CommandExecutionException + + +class TestError(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.error_msg = "error message" + self.error = Error(pattern="test pattern", error=self.error_msg) + + def test_call(self): + """Check that method will raise CommandExecutionException""" + with self.assertRaisesRegex(CommandExecutionException, self.error_msg): + self.error() + + @mock.patch("cloudshell.cli.service.error_map.re") + def test_match_return_true(self, re): + """Check that method will return True if output matches pattern""" + re.search.return_value = True + output = "test output" + # act + result = self.error.match(output) + # verify + self.assertTrue(result) + + @mock.patch("cloudshell.cli.service.error_map.re") + def test_match_return_false(self, re): + """Check that method will return False if output doesn't match pattern""" + re.search.return_value = False + output = "test output" + # act + result = self.error.match(output) + # verify + self.assertFalse(result) + + +class TestErrorMap(unittest.TestCase): + def setUp(self): + self.logger = mock.MagicMock() + self.errors = [Error(pattern="[Pp]attern 1", error="error 1"), + Error(pattern="[Pp]attern 2", error="error 2")] + + self.error_map = ErrorMap(errors=self.errors) + + def test_errors(self): + """Check that method will return errors""" + # verify + self.assertEqual(self.error_map.errors, self.errors) + + def test_extend(self): + """Check that extend will add new errors and will not override existing one""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + self.error_map.extend(error_map=default_error_map) + # verify + self.assertEqual(self.error_map.errors, [error1, error2, default_error1]) + + def test_extend_with_override_true(self): + """Check that extend will add new errors and will not override existing one""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + self.error_map.extend(error_map=default_error_map, override=True) + # verify + self.assertEqual(self.error_map.errors, [error1, default_error2, default_error1]) + + def test_add(self): + """Check that __add__ method will create new ErrorMap""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + result = self.error_map + default_error_map + # verify + self.assertEqual(result.errors, [error1, error2, default_error1]) + From 0b35ecef33d849035c2c51b1166547bddb7a9d3f Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 17:52:51 +0300 Subject: [PATCH 14/59] use compiled re pattern in the "error_map" and "action_map" modules --- cloudshell/cli/service/action_map.py | 3 ++- cloudshell/cli/service/error_map.py | 3 ++- tests/cli/service/test_action_map.py | 12 ++++-------- tests/cli/service/test_error_map.py | 12 ++++-------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index f210f40..2eb207e 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -13,6 +13,7 @@ def __init__(self, pattern, callback, execute_once=False): :param bool execute_once: """ self.pattern = pattern + self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.callback = callback self.execute_once = execute_once @@ -38,7 +39,7 @@ def match(self, output): :param str output: :rtype: bool """ - return bool(re.search(self.pattern, output, re.DOTALL)) + return bool(self.compiled_pattern.search(output)) class ActionMap: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 16e2fe3..4b55e08 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -12,6 +12,7 @@ def __init__(self, pattern, error): :param str|CommandExecutionException error: """ self.pattern = pattern + self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.error = error def __call__(self): @@ -37,7 +38,7 @@ def match(self, output): :param str output: :rtype: bool """ - return bool(re.search(self.pattern, output, re.DOTALL)) + return bool(self.compiled_pattern.search(output)) class ErrorMap: diff --git a/tests/cli/service/test_action_map.py b/tests/cli/service/test_action_map.py index ad72e56..8f849b0 100644 --- a/tests/cli/service/test_action_map.py +++ b/tests/cli/service/test_action_map.py @@ -19,21 +19,17 @@ def test_call(self): # verify self.callback.assert_called_once_with(self.session, self.logger) - @mock.patch("cloudshell.cli.service.action_map.re") - def test_match_return_true(self, re): + def test_match_return_true(self): """Check that method will return True if output matches pattern""" - re.search.return_value = True - output = "test output" + output = "test pattern" # act result = self.action.match(output) # verify self.assertTrue(result) - @mock.patch("cloudshell.cli.service.action_map.re") - def test_match_return_false(self, re): + def test_match_return_false(self): """Check that method will return False if output doesn't match pattern""" - re.search.return_value = False - output = "test output" + output = "missed pattern" # act result = self.action.match(output) # verify diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py index aa9b89e..2f2f425 100644 --- a/tests/cli/service/test_error_map.py +++ b/tests/cli/service/test_error_map.py @@ -18,21 +18,17 @@ def test_call(self): with self.assertRaisesRegex(CommandExecutionException, self.error_msg): self.error() - @mock.patch("cloudshell.cli.service.error_map.re") - def test_match_return_true(self, re): + def test_match_return_true(self): """Check that method will return True if output matches pattern""" - re.search.return_value = True - output = "test output" + output = "test pattern" # act result = self.error.match(output) # verify self.assertTrue(result) - @mock.patch("cloudshell.cli.service.error_map.re") - def test_match_return_false(self, re): + def test_match_return_false(self): """Check that method will return False if output doesn't match pattern""" - re.search.return_value = False - output = "test output" + output = "missed pattern" # act result = self.error.match(output) # verify From 49b4641f468c88e721d95b0bef9d74c9cda71c5f Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 17:57:35 +0300 Subject: [PATCH 15/59] move "__call__" function to "process" in ActionMap and ErrorMap classes --- cloudshell/cli/service/action_map.py | 2 +- cloudshell/cli/service/error_map.py | 2 +- cloudshell/cli/session/expect_session.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 2eb207e..d5a4c23 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -93,7 +93,7 @@ def extend(self, action_map, override=False): self.matched_patterns |= action_map.matched_patterns - def __call__(self, session, logger, output, check_action_loop_detector, action_loop_detector): + def process(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ :param cloudshell.cli.session.expect_session.ExpectSession session: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 4b55e08..6bf1bb1 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -80,7 +80,7 @@ def extend(self, error_map, override=False): continue self.add(error) - def __call__(self, output, logger): + def process(self, output, logger): """ :param str output: diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index b0124e6..b054eb1 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -258,11 +258,11 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err output_list.append(output_str) is_correct_exit = True - action_matched = action_map(session=self, - logger=logger, - output=output_str, - check_action_loop_detector=check_action_loop_detector, - action_loop_detector=action_loop_detector) + action_matched = action_map.process(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=check_action_loop_detector, + action_loop_detector=action_loop_detector) if action_matched: output_list.append(output_str) @@ -276,8 +276,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err 'Session Loop limit exceeded, {} loops'.format(retries_count)) result_output = ''.join(output_list) - - error_map(output=result_output, logger=logger) + error_map.process(output=result_output, logger=logger) # Read buffer to the end. Useful when expected_string isn't last in buffer result_output += self._clear_buffer(self._clear_buffer_timeout, logger) From 4cdf9f7858ee39df3f5835ba11afd84bc6c9555b Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:19:11 +0300 Subject: [PATCH 16/59] simplify ActionMap and ErrorMap "__add__" methods --- cloudshell/cli/service/action_map.py | 16 +++++++++------- cloudshell/cli/service/error_map.py | 10 +++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index d5a4c23..114e3e4 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -79,11 +79,12 @@ def add(self, action): """ self._actions_dict[action.pattern] = action - def extend(self, action_map, override=False): + def extend(self, action_map, override=False, extend_matched_patterns=True): """ :param ActionMap action_map: :param bool override: + :param bool extend_matched_patterns: :return: """ for action in action_map.actions: @@ -91,7 +92,8 @@ def extend(self, action_map, override=False): continue self.add(action) - self.matched_patterns |= action_map.matched_patterns + if extend_matched_patterns: + self.matched_patterns |= action_map.matched_patterns def process(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ @@ -126,11 +128,11 @@ def __add__(self, other): :param other: :rtype: ActionMap """ - if isinstance(other, type(self)): - actions = self.actions + [action for action in other.actions if action.pattern not in - [action.pattern for action in self.actions]] - - return ActionMap(actions=actions) + action_map_class = type(self) + if isinstance(other, action_map_class): + action_map = action_map_class(actions=self.actions) + action_map.extend(other, extend_matched_patterns=False) + return action_map raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 6bf1bb1..ba6857d 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -99,11 +99,11 @@ def __add__(self, other): :param other: :rtype: ActionMap """ - if isinstance(other, type(self)): - errors = self.errors + [error for error in other.errors if error.pattern not in - [error.pattern for error in self.errors]] - - return ErrorMap(errors=errors) + error_map_class = type(self) + if isinstance(other, error_map_class): + error_map = error_map_class(errors=self.errors) + error_map.extend(other) + return error_map raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") From 502c994e7668c5f79b7f0881ef1ca5e6cfc55918 Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:48:02 +0300 Subject: [PATCH 17/59] pass "output" argument to the ErrorMap "__call__" method --- cloudshell/cli/service/error_map.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index ba6857d..5dc82c0 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -15,9 +15,10 @@ def __init__(self, pattern, error): self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.error = error - def __call__(self): + def __call__(self, output): """ + :param str output: :raises: CommandExecutionException """ if isinstance(self.error, CommandExecutionException): @@ -91,7 +92,7 @@ def process(self, output, logger): for error in self.errors: if error.match(output): logger.debug(f"Matched Error with pattern: {error.pattern}") - error() + error(output) def __add__(self, other): """ From eceb263490a1d2ed8e1b2b13b3a74ba4a13f3672 Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:52:02 +0300 Subject: [PATCH 18/59] fix unit test for the ErrorMap class --- tests/cli/service/test_error_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py index 2f2f425..abb9640 100644 --- a/tests/cli/service/test_error_map.py +++ b/tests/cli/service/test_error_map.py @@ -16,7 +16,7 @@ def setUp(self): def test_call(self): """Check that method will raise CommandExecutionException""" with self.assertRaisesRegex(CommandExecutionException, self.error_msg): - self.error() + self.error(output="test output") def test_match_return_true(self): """Check that method will return True if output matches pattern""" From dab9bb2ffb3fce5255d196c9911fa165856a0932 Mon Sep 17 00:00:00 2001 From: anthony Date: Wed, 3 Jul 2019 15:25:52 +0300 Subject: [PATCH 19/59] update docstrings and error messages --- cloudshell/cli/session/expect_session.py | 146 +++++++++++------------ 1 file changed, 70 insertions(+), 76 deletions(-) diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index b054eb1..fe6aea0 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -12,12 +12,9 @@ class ExpectSession(Session, metaclass=ABCMeta): - """ - Help to handle additional actions during send command - """ + """Help to handle additional actions during send command""" SESSION_TYPE = 'EXPECT' - MAX_LOOP_RETRIES = 20 READ_TIMEOUT = 30 EMPTY_LOOP_TIMEOUT = 0.5 @@ -30,19 +27,17 @@ def __init__(self, timeout=READ_TIMEOUT, new_line='\r', max_loop_retries=MAX_LOO empty_loop_timeout=EMPTY_LOOP_TIMEOUT, loop_detector_max_action_loops=LOOP_DETECTOR_MAX_ACTION_LOOPS, loop_detector_max_combination_length=LOOP_DETECTOR_MAX_COMBINATION_LENGTH, clear_buffer_timeout=CLEAR_BUFFER_TIMEOUT, reconnect_timeout=RECONNECT_TIMEOUT): - """ - :param timeout: - :param new_line: - :param max_loop_retries: - :param empty_loop_timeout: - :param loop_detector_max_action_loops: - :param loop_detector_max_combination_length: - :param clear_buffer_timeout: + :param int timeout: + :param str new_line: + :param int max_loop_retries: + :param float empty_loop_timeout: + :param int loop_detector_max_action_loops: + :param int loop_detector_max_combination_length: + :param float clear_buffer_timeout: :return: """ - self._new_line = new_line self._timeout = timeout self._max_loop_retries = max_loop_retries @@ -63,32 +58,42 @@ def session_type(self): @abstractmethod def _connect_actions(self, prompt, logger): """Read out buffer and run on_session_start actions - :param prompt: expected string in output - :param logger: logger - """ + :param str prompt: expected string in output + :param logging.Logger logger: logger + """ pass @abstractmethod def _initialize_session(self, prompt, logger): """Create handler and initialize session - :param prompt: expected string in output - :param logger: logger + + :param str prompt: expected string in output + :param logging.Logger logger: logger """ pass def set_active(self, state): + """ + + :param bool state: + :return: + """ self._active = state def active(self): + """ + + :rtype: bool + """ return self._active def _clear_buffer(self, timeout, logger): - """ - Clear buffer + """Clear buffer - :param timeout: - :return: + :param int|float timeout: + :param logging.Logger logger: + :rtype: str """ out = '' while True: @@ -103,11 +108,11 @@ def _clear_buffer(self, timeout, logger): return out def connect(self, prompt, logger): - """Connect to device. - :param prompt: expected string in output - :param logger: logger - """ + """Connect to device + :param str prompt: expected string in output + :param logging.Logger logger: logger + """ try: self._initialize_session(prompt, logger) self._connect_actions(prompt, logger) @@ -117,26 +122,25 @@ def connect(self, prompt, logger): raise def send_line(self, command, logger): - """ - Add new line to the end of command string and send + """Add new line to the end of command string and send - :param command: + :param str command: + :param logging.Logger logger: :return: """ self._send(command + self._new_line, logger) def _receive_all(self, timeout, logger): - """ - Read as much as possible before catch SessionTimeoutException - :param timeout: - :param logger: - :return: + """Read as much as possible before catch SessionTimeoutException + + :param int timeout: + :param logging.Logger logger: :rtype: str """ - if not timeout: - timeout = self._timeout + timeout = timeout or self._timeout start_time = time.time() read_buffer = '' + while True: try: read_buffer += self._receive(0.1, logger) @@ -144,41 +148,37 @@ def _receive_all(self, timeout, logger): if read_buffer: return read_buffer elif time.time() - start_time > timeout: - raise ExpectedSessionException(self.__class__.__name__, 'Socket closed by timeout') + raise ExpectedSessionException('Socket closed by timeout') def _generate_command_pattern(self, command): - """ - Generate command_pattern - :param command: + """Generate command_pattern + + :param str command: :return: """ if command not in self._command_patterns: self._command_patterns[command] = '\s*' + re.sub(r'\\\s+', '\s+', re.escape(command)) + '\s*' + return self._command_patterns[command] def probe_for_prompt(self, expected_string, logger): - """ - Matched string for regexp - :param expected_string: - :param logger: + """Matched string for regexp + + :param str expected_string: + :param logging.Logger logger: :return: """ return self.hardware_expect('', expected_string, logger) def match_prompt(self, prompt, match_string, logger): - """ - Main verification for the prompt match - :param prompt: Expected string, string or regular expression - :type prompt: str - :param match_string: Match string - :type match_string: str - :param logger + """Main verification for the prompt match + + :param str prompt: expected string, string or regular expression + :param str match_string: Match string + :param logging.Logger logger: :rtype: bool """ - if re.search(prompt, match_string, re.DOTALL): - return True - else: - return False + return bool(re.search(prompt, match_string, re.DOTALL)) def hardware_expect(self, command, expected_string, logger, action_map=None, error_map=None, timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, @@ -195,27 +195,23 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err :param int timeout: session timeout :param int retries: maximal retries count :param bool check_action_loop_detector: + :param bool empty_loop_timeout: :param bool remove_command_from_output: In some switches the output string includes the command which was called. The flag used to verify whether the the command string removed from the output string. :rtype: str """ - if not action_map: - action_map = ActionMap() - - if not error_map: - error_map = ErrorMap() - + action_map = action_map or ActionMap() + error_map = error_map or ErrorMap() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout if command is not None: self._clear_buffer(self._clear_buffer_timeout, logger) - - logger.debug('Command: {}'.format(command)) + logger.debug(f'Command: {command}') self.send_line(command, logger) if not expected_string: - raise ExpectedSessionException(self.__class__.__name__, 'List of expected messages can\'t be empty!') + raise ExpectedSessionException('List of expected messages can\'t be empty!') # Loop until one of the expressions is matched or MAX_RETRIES # nothing is expected (usually used for exit) @@ -272,8 +268,7 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err break if not is_correct_exit: - raise SessionLoopLimitException(self.__class__.__name__, - 'Session Loop limit exceeded, {} loops'.format(retries_count)) + raise SessionLoopLimitException(f'Session Loop limit exceeded, {retries_count} loops') result_output = ''.join(output_list) error_map.process(output=result_output, logger=logger) @@ -283,23 +278,22 @@ def hardware_expect(self, command, expected_string, logger, action_map=None, err return result_output def reconnect(self, prompt, logger, timeout=None): - """ - Recconnect implementation + """Reconnect implementation - :param prompt: - :param logger: - :param timeout: + :param str prompt: + :param logging.Logger logger: + :param int timeout: :return: """ logger.debug('Reconnect') timeout = timeout or self._reconnect_timeout - call_time = time.time() + while time.time() - call_time < timeout: try: self.disconnect() return self.connect(prompt, logger) - except Exception as e: - logger.debug(e) - raise ExpectedSessionException(self.__class__.__name__, - 'Reconnect unsuccessful, timeout exceeded, see logs for more details') + except Exception: + logger.debug('Failed to reconnect:', exc_info=True) + + raise ExpectedSessionException('Reconnect unsuccessful, timeout exceeded, see logs for more details') From af6040346678bb9cf6532266e7fa5542f3ce74a0 Mon Sep 17 00:00:00 2001 From: Anthony Date: Thu, 11 Jul 2019 16:38:19 +0300 Subject: [PATCH 20/59] rename ExcpectSession to AbstractSession --- cloudshell/cli/service/command_mode_helper.py | 1 - cloudshell/cli/service/session_pool.py | 7 +- .../service/session_pool_context_manager.py | 4 +- .../cli/service/session_pool_manager.py | 3 +- cloudshell/cli/session/expect_session.py | 299 ----------------- cloudshell/cli/session/session.py | 309 ++++++++++++++++-- cloudshell/cli/session/ssh_session.py | 8 +- cloudshell/cli/session/tcp_session.py | 8 +- cloudshell/cli/session/telnet_session.py | 8 +- cloudshell/cli/session/tl1_session.py | 2 +- ...ct_session.py => test_abstract_session.py} | 81 +++-- tests/cli/session/test_ssh_session.py | 4 +- tests/cli/session/test_telnet_session.py | 4 +- .../cli/test_session_pool_context_manager.py | 2 +- 14 files changed, 349 insertions(+), 391 deletions(-) delete mode 100644 cloudshell/cli/session/expect_session.py rename tests/cli/session/{test_expect_session.py => test_abstract_session.py} (79%) diff --git a/cloudshell/cli/service/command_mode_helper.py b/cloudshell/cli/service/command_mode_helper.py index 860d370..9bf57d1 100644 --- a/cloudshell/cli/service/command_mode_helper.py +++ b/cloudshell/cli/service/command_mode_helper.py @@ -2,7 +2,6 @@ from cloudshell.cli.service.command_mode import CommandMode, CommandModeException from cloudshell.cli.service.node import NodeOperations -from cloudshell.cli.session.session import Session from functools import reduce diff --git a/cloudshell/cli/service/session_pool.py b/cloudshell/cli/service/session_pool.py index 4c24d2f..6d814f5 100644 --- a/cloudshell/cli/service/session_pool.py +++ b/cloudshell/cli/service/session_pool.py @@ -1,13 +1,12 @@ from abc import ABCMeta, abstractmethod -from cloudshell.cli.session.session import Session class SessionPool(object, metaclass=ABCMeta): @abstractmethod def get_session(self, new_sessions, prompt, logger): - """ - Get session from pool - :rtype Session + """Get session from pool + + :rtype: cloudshell.cli.session.session.AbstractSession """ pass diff --git a/cloudshell/cli/service/session_pool_context_manager.py b/cloudshell/cli/service/session_pool_context_manager.py index 9f02894..ed53dac 100644 --- a/cloudshell/cli/service/session_pool_context_manager.py +++ b/cloudshell/cli/service/session_pool_context_manager.py @@ -1,6 +1,6 @@ from cloudshell.cli.service.cli_service_impl import CliServiceImpl as CliService from cloudshell.cli.service.command_mode_helper import CommandModeHelper -from cloudshell.cli.session.expect_session import CommandExecutionException +from cloudshell.cli.session.session_exceptions import CommandExecutionException class SessionPoolContextManager(object): @@ -53,7 +53,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if self._active_session: - if exc_type and not issubclass(exc_type, self.IGNORED_EXCEPTIONS) or not self._active_session.active(): + if exc_type and not issubclass(exc_type, self.IGNORED_EXCEPTIONS) or not self._active_session.active: self._session_pool.remove_session(self._active_session, self._logger) else: self._session_pool.return_session(self._active_session, self._logger) diff --git a/cloudshell/cli/service/session_pool_manager.py b/cloudshell/cli/service/session_pool_manager.py index bc22139..8022c94 100644 --- a/cloudshell/cli/service/session_pool_manager.py +++ b/cloudshell/cli/service/session_pool_manager.py @@ -4,7 +4,6 @@ from cloudshell.cli.service.cli_exception import CliException from cloudshell.cli.service.session_pool import SessionPool -from cloudshell.cli.session.session import Session from cloudshell.cli.service.session_manager_impl import SessionManagerImpl as SessionManager @@ -48,7 +47,7 @@ def get_session(self, defined_sessions, prompt, logger): :param prompt: :param logger: :return: - :rtype: Session + :rtype: cloudshell.cli.session.session.AbstractSession """ call_time = time.time() diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py deleted file mode 100644 index fe6aea0..0000000 --- a/cloudshell/cli/session/expect_session.py +++ /dev/null @@ -1,299 +0,0 @@ -import re -import time -from abc import ABCMeta, abstractmethod - -from cloudshell.cli.service.action_map import ActionMap -from cloudshell.cli.service.error_map import ErrorMap -from cloudshell.cli.service.action_map import ActionLoopDetector -from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer -from cloudshell.cli.session.session import Session -from cloudshell.cli.session.session_exceptions import SessionLoopLimitException, \ - ExpectedSessionException, CommandExecutionException, SessionReadTimeout, SessionReadEmptyData - - -class ExpectSession(Session, metaclass=ABCMeta): - """Help to handle additional actions during send command""" - - SESSION_TYPE = 'EXPECT' - MAX_LOOP_RETRIES = 20 - READ_TIMEOUT = 30 - EMPTY_LOOP_TIMEOUT = 0.5 - CLEAR_BUFFER_TIMEOUT = 0.1 - LOOP_DETECTOR_MAX_ACTION_LOOPS = 3 - LOOP_DETECTOR_MAX_COMBINATION_LENGTH = 4 - RECONNECT_TIMEOUT = 30 - - def __init__(self, timeout=READ_TIMEOUT, new_line='\r', max_loop_retries=MAX_LOOP_RETRIES, - empty_loop_timeout=EMPTY_LOOP_TIMEOUT, loop_detector_max_action_loops=LOOP_DETECTOR_MAX_ACTION_LOOPS, - loop_detector_max_combination_length=LOOP_DETECTOR_MAX_COMBINATION_LENGTH, - clear_buffer_timeout=CLEAR_BUFFER_TIMEOUT, reconnect_timeout=RECONNECT_TIMEOUT): - """ - - :param int timeout: - :param str new_line: - :param int max_loop_retries: - :param float empty_loop_timeout: - :param int loop_detector_max_action_loops: - :param int loop_detector_max_combination_length: - :param float clear_buffer_timeout: - :return: - """ - self._new_line = new_line - self._timeout = timeout - self._max_loop_retries = max_loop_retries - self._empty_loop_timeout = empty_loop_timeout - - self._loop_detector_max_action_loops = loop_detector_max_action_loops - self._loop_detector_max_combination_length = loop_detector_max_combination_length - self._clear_buffer_timeout = clear_buffer_timeout - self._reconnect_timeout = reconnect_timeout - - self._active = False - self._command_patterns = {} - - @property - def session_type(self): - return self.SESSION_TYPE - - @abstractmethod - def _connect_actions(self, prompt, logger): - """Read out buffer and run on_session_start actions - - :param str prompt: expected string in output - :param logging.Logger logger: logger - """ - pass - - @abstractmethod - def _initialize_session(self, prompt, logger): - """Create handler and initialize session - - :param str prompt: expected string in output - :param logging.Logger logger: logger - """ - pass - - def set_active(self, state): - """ - - :param bool state: - :return: - """ - self._active = state - - def active(self): - """ - - :rtype: bool - """ - return self._active - - def _clear_buffer(self, timeout, logger): - """Clear buffer - - :param int|float timeout: - :param logging.Logger logger: - :rtype: str - """ - out = '' - while True: - try: - read_buffer = self._receive(timeout, logger) - except (SessionReadTimeout, SessionReadEmptyData): - read_buffer = None - if read_buffer: - out += read_buffer - else: - break - return out - - def connect(self, prompt, logger): - """Connect to device - - :param str prompt: expected string in output - :param logging.Logger logger: logger - """ - try: - self._initialize_session(prompt, logger) - self._connect_actions(prompt, logger) - self.set_active(True) - except: - self.disconnect() - raise - - def send_line(self, command, logger): - """Add new line to the end of command string and send - - :param str command: - :param logging.Logger logger: - :return: - """ - self._send(command + self._new_line, logger) - - def _receive_all(self, timeout, logger): - """Read as much as possible before catch SessionTimeoutException - - :param int timeout: - :param logging.Logger logger: - :rtype: str - """ - timeout = timeout or self._timeout - start_time = time.time() - read_buffer = '' - - while True: - try: - read_buffer += self._receive(0.1, logger) - except (SessionReadTimeout, SessionReadEmptyData): - if read_buffer: - return read_buffer - elif time.time() - start_time > timeout: - raise ExpectedSessionException('Socket closed by timeout') - - def _generate_command_pattern(self, command): - """Generate command_pattern - - :param str command: - :return: - """ - if command not in self._command_patterns: - self._command_patterns[command] = '\s*' + re.sub(r'\\\s+', '\s+', re.escape(command)) + '\s*' - - return self._command_patterns[command] - - def probe_for_prompt(self, expected_string, logger): - """Matched string for regexp - - :param str expected_string: - :param logging.Logger logger: - :return: - """ - return self.hardware_expect('', expected_string, logger) - - def match_prompt(self, prompt, match_string, logger): - """Main verification for the prompt match - - :param str prompt: expected string, string or regular expression - :param str match_string: Match string - :param logging.Logger logger: - :rtype: bool - """ - return bool(re.search(prompt, match_string, re.DOTALL)) - - def hardware_expect(self, command, expected_string, logger, action_map=None, error_map=None, - timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, - remove_command_from_output=True, **optional_args): - """Get response form the device and compare it to action_map, error_map and expected_string patterns, - - perform actions specified in action_map if any, and return output. - Raise Exception if receive empty response from device within a minute - :param str command: command to send - :param str expected_string: expected string - :param logging.Logger logger: logger - :param cloudshell.cli.service.action_map.ActionMap action_map: - :param cloudshell.cli.service.error_map.ErrorMap error_map: - :param int timeout: session timeout - :param int retries: maximal retries count - :param bool check_action_loop_detector: - :param bool empty_loop_timeout: - :param bool remove_command_from_output: In some switches the output string includes the command which was - called. The flag used to verify whether the the command string removed from the output string. - :rtype: str - """ - action_map = action_map or ActionMap() - error_map = error_map or ErrorMap() - retries = retries or self._max_loop_retries - empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout - - if command is not None: - self._clear_buffer(self._clear_buffer_timeout, logger) - logger.debug(f'Command: {command}') - self.send_line(command, logger) - - if not expected_string: - raise ExpectedSessionException('List of expected messages can\'t be empty!') - - # Loop until one of the expressions is matched or MAX_RETRIES - # nothing is expected (usually used for exit) - output_list = list() - output_str = '' - retries_count = 0 - is_correct_exit = False - - action_loop_detector = ActionLoopDetector(self._loop_detector_max_action_loops, - self._loop_detector_max_combination_length) - - while retries == 0 or retries_count < retries: - - # try: - # read_buffer = self._receive(timeout, logger) - # read all data from buffer - read_buffer = self._receive_all(timeout, logger) - # except socket.timeout: - # read_buffer = None - - if read_buffer: - read_buffer = normalize_buffer(read_buffer) - logger.debug(read_buffer) - output_str += read_buffer - # if option remove_command_from_output is set to True, look for command in output buffer, - # remove it in case of found - if command and remove_command_from_output: - command_pattern = self._generate_command_pattern(command) - if re.search(command_pattern, output_str, flags=re.MULTILINE): - output_str = re.sub(command_pattern, '', output_str, count=1, flags=re.MULTILINE) - remove_command_from_output = False - retries_count = 0 - else: - retries_count += 1 - time.sleep(empty_loop_timeout) - continue - - if self.match_prompt(expected_string, output_str, logger): - # logger.debug('Expected str: {}'.format(expected_string)) - output_list.append(output_str) - is_correct_exit = True - - action_matched = action_map.process(session=self, - logger=logger, - output=output_str, - check_action_loop_detector=check_action_loop_detector, - action_loop_detector=action_loop_detector) - - if action_matched: - output_list.append(output_str) - output_str = '' - - if is_correct_exit: - break - - if not is_correct_exit: - raise SessionLoopLimitException(f'Session Loop limit exceeded, {retries_count} loops') - - result_output = ''.join(output_list) - error_map.process(output=result_output, logger=logger) - - # Read buffer to the end. Useful when expected_string isn't last in buffer - result_output += self._clear_buffer(self._clear_buffer_timeout, logger) - return result_output - - def reconnect(self, prompt, logger, timeout=None): - """Reconnect implementation - - :param str prompt: - :param logging.Logger logger: - :param int timeout: - :return: - """ - logger.debug('Reconnect') - timeout = timeout or self._reconnect_timeout - call_time = time.time() - - while time.time() - call_time < timeout: - try: - self.disconnect() - return self.connect(prompt, logger) - except Exception: - logger.debug('Failed to reconnect:', exc_info=True) - - raise ExpectedSessionException('Reconnect unsuccessful, timeout exceeded, see logs for more details') diff --git a/cloudshell/cli/session/session.py b/cloudshell/cli/session/session.py index 05639ac..d0b9eed 100644 --- a/cloudshell/cli/session/session.py +++ b/cloudshell/cli/session/session.py @@ -1,50 +1,311 @@ +import re +import time from abc import ABCMeta, abstractmethod -from collections import OrderedDict +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap +from cloudshell.cli.service.action_map import ActionLoopDetector +from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer +from cloudshell.cli.session.session_exceptions import SessionLoopLimitException, \ + ExpectedSessionException, CommandExecutionException, SessionReadTimeout, SessionReadEmptyData -class Session(object, metaclass=ABCMeta): - @abstractmethod - def connect(self, prompt, logger): - pass - @abstractmethod - def disconnect(self): - pass +class AbstractSession(metaclass=ABCMeta): + SESSION_TYPE = 'EXPECT' + MAX_LOOP_RETRIES = 20 + READ_TIMEOUT = 30 + EMPTY_LOOP_TIMEOUT = 0.5 + CLEAR_BUFFER_TIMEOUT = 0.1 + LOOP_DETECTOR_MAX_ACTION_LOOPS = 3 + LOOP_DETECTOR_MAX_COMBINATION_LENGTH = 4 + RECONNECT_TIMEOUT = 30 @abstractmethod def _send(self, command, logger): pass @abstractmethod - def send_line(self, command, logger): + def _receive(self, timeout, logger): pass @abstractmethod - def _receive(self, timeout, logger): + def disconnect(self): pass @abstractmethod - def hardware_expect(self, command, expected_string, logger, action_map=OrderedDict(), error_map=OrderedDict(), - timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, - **optional_args): + def _connect_actions(self, prompt, logger): + """Read out buffer and run on_session_start actions + + :param str prompt: expected string in output + :param logging.Logger logger: logger + """ pass @abstractmethod - def probe_for_prompt(self, expected_string, logger): + def _initialize_session(self, prompt, logger): + """Create handler and initialize session + + :param str prompt: expected string in output + :param logging.Logger logger: logger + """ pass - @abstractmethod + @property + def session_type(self): + return self.SESSION_TYPE + + @property + def active(self): + """ + + :rtype: bool + """ + return self._active + + @active.setter + def active(self, val): + """ + + :param bool val: + :return: + """ + self._active = val + + def __init__(self, timeout=READ_TIMEOUT, new_line='\r', max_loop_retries=MAX_LOOP_RETRIES, + empty_loop_timeout=EMPTY_LOOP_TIMEOUT, loop_detector_max_action_loops=LOOP_DETECTOR_MAX_ACTION_LOOPS, + loop_detector_max_combination_length=LOOP_DETECTOR_MAX_COMBINATION_LENGTH, + clear_buffer_timeout=CLEAR_BUFFER_TIMEOUT, reconnect_timeout=RECONNECT_TIMEOUT): + """ + + :param int timeout: + :param str new_line: + :param int max_loop_retries: + :param float empty_loop_timeout: + :param int loop_detector_max_action_loops: + :param int loop_detector_max_combination_length: + :param float clear_buffer_timeout: + :return: + """ + self._new_line = new_line + self._timeout = timeout + self._max_loop_retries = max_loop_retries + self._empty_loop_timeout = empty_loop_timeout + + self._loop_detector_max_action_loops = loop_detector_max_action_loops + self._loop_detector_max_combination_length = loop_detector_max_combination_length + self._clear_buffer_timeout = clear_buffer_timeout + self._reconnect_timeout = reconnect_timeout + + self._active = False + self._command_patterns = {} + + def _clear_buffer(self, timeout, logger): + """Clear buffer + + :param int|float timeout: + :param logging.Logger logger: + :rtype: str + """ + out = '' + while True: + try: + read_buffer = self._receive(timeout, logger) + except (SessionReadTimeout, SessionReadEmptyData): + read_buffer = None + if read_buffer: + out += read_buffer + else: + break + return out + + def connect(self, prompt, logger): + """Connect to device + + :param str prompt: expected string in output + :param logging.Logger logger: logger + """ + try: + self._initialize_session(prompt, logger) + self._connect_actions(prompt, logger) + self.active = True + except Exception: + self.disconnect() + raise + + def send_line(self, command, logger): + """Add new line to the end of command string and send + + :param str command: + :param logging.Logger logger: + :return: + """ + self._send(command + self._new_line, logger) + + def _receive_all(self, timeout, logger): + """Read as much as possible before catch SessionTimeoutException + + :param int timeout: + :param logging.Logger logger: + :rtype: str + """ + timeout = timeout or self._timeout + start_time = time.time() + read_buffer = '' + + while True: + try: + read_buffer += self._receive(0.1, logger) + except (SessionReadTimeout, SessionReadEmptyData): + if read_buffer: + return read_buffer + elif time.time() - start_time > timeout: + raise ExpectedSessionException('Socket closed by timeout') + + def _generate_command_pattern(self, command): + """Generate command_pattern + + :param str command: + :return: + """ + if command not in self._command_patterns: + self._command_patterns[command] = '\s*' + re.sub(r'\\\s+', '\s+', re.escape(command)) + '\s*' + + return self._command_patterns[command] + + def probe_for_prompt(self, expected_string, logger): + """Matched string for regexp + + :param str expected_string: + :param logging.Logger logger: + :return: + """ + return self.hardware_expect('', expected_string, logger) + def match_prompt(self, prompt, match_string, logger): - pass + """Main verification for the prompt match + + :param str prompt: expected string, string or regular expression + :param str match_string: Match string + :param logging.Logger logger: + :rtype: bool + """ + return bool(re.search(prompt, match_string, re.DOTALL)) + + def hardware_expect(self, command, expected_string, logger, action_map=None, error_map=None, + timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, + remove_command_from_output=True): + """Get response form the device and compare it to action_map, error_map and expected_string patterns, + + perform actions specified in action_map if any, and return output. + Raise Exception if receive empty response from device within a minute + :param str command: command to send + :param str expected_string: expected string + :param logging.Logger logger: logger + :param collections.OrderedDict action_map: dict with {re_str: action} to trigger some action on received string + :param error_map: expected error map with subclass of CommandExecutionException or str + :type error_map: dict[str, CommandExecutionException|str] + :param int timeout: session timeout + :param int retries: maximal retries count + :param bool check_action_loop_detector: + :param bool empty_loop_timeout: + :param bool remove_command_from_output: In some switches the output string includes the command which was called. + The flag used to verify whether the the command string removed from the output string. + :rtype: str + """ + action_map = action_map or ActionMap() + error_map = error_map or ErrorMap() + retries = retries or self._max_loop_retries + empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout + + if command is not None: + self._clear_buffer(self._clear_buffer_timeout, logger) + logger.debug(f'Command: {command}') + self.send_line(command, logger) + + if not expected_string: + raise ExpectedSessionException('List of expected messages can\'t be empty!') + + # Loop until one of the expressions is matched or MAX_RETRIES + # nothing is expected (usually used for exit) + output_list = list() + output_str = '' + retries_count = 0 + is_correct_exit = False + + action_loop_detector = ActionLoopDetector(self._loop_detector_max_action_loops, + self._loop_detector_max_combination_length) + + while retries == 0 or retries_count < retries: + + # try: + # read_buffer = self._receive(timeout, logger) + # read all data from buffer + read_buffer = self._receive_all(timeout, logger) + # except socket.timeout: + # read_buffer = None + + if read_buffer: + read_buffer = normalize_buffer(read_buffer) + logger.debug(read_buffer) + output_str += read_buffer + # if option remove_command_from_output is set to True, look for command in output buffer, + # remove it in case of found + if command and remove_command_from_output: + command_pattern = self._generate_command_pattern(command) + if re.search(command_pattern, output_str, flags=re.MULTILINE): + output_str = re.sub(command_pattern, '', output_str, count=1, flags=re.MULTILINE) + remove_command_from_output = False + retries_count = 0 + else: + retries_count += 1 + time.sleep(empty_loop_timeout) + continue + + if self.match_prompt(expected_string, output_str, logger): + # logger.debug('Expected str: {}'.format(expected_string)) + output_list.append(output_str) + is_correct_exit = True + + action_matched = action_map.process(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=check_action_loop_detector, + action_loop_detector=action_loop_detector) + + if action_matched: + output_list.append(output_str) + output_str = '' + + if is_correct_exit: + break + + if not is_correct_exit: + raise SessionLoopLimitException(f'Session Loop limit exceeded, {retries_count} loops') + + result_output = ''.join(output_list) + error_map.process(output=result_output, logger=logger) + + # Read buffer to the end. Useful when expected_string isn't last in buffer + result_output += self._clear_buffer(self._clear_buffer_timeout, logger) + return result_output - @abstractmethod def reconnect(self, prompt, logger, timeout=None): - pass + """Reconnect implementation - @abstractmethod - def active(self): - pass + :param str prompt: + :param logging.Logger logger: + :param int timeout: + :return: + """ + logger.debug('Reconnect') + timeout = timeout or self._reconnect_timeout + call_time = time.time() - @abstractmethod - def set_active(self, state): - pass + while time.time() - call_time < timeout: + try: + self.disconnect() + return self.connect(prompt, logger) + except Exception: + logger.debug('Failed to reconnect:', exc_info=True) + + raise ExpectedSessionException('Reconnect unsuccessful, timeout exceeded, see logs for more details') diff --git a/cloudshell/cli/session/ssh_session.py b/cloudshell/cli/session/ssh_session.py index 8fc8f8c..3c21c1c 100644 --- a/cloudshell/cli/session/ssh_session.py +++ b/cloudshell/cli/session/ssh_session.py @@ -5,20 +5,20 @@ from cloudshell.cli.session.session_exceptions import SessionException, SessionReadTimeout, SessionReadEmptyData from cloudshell.cli.session.connection_params import ConnectionParams -from cloudshell.cli.session.expect_session import ExpectSession +from cloudshell.cli.session.session import AbstractSession class SSHSessionException(SessionException): pass -class SSHSession(ExpectSession, ConnectionParams): +class SSHSession(AbstractSession, ConnectionParams): SESSION_TYPE = 'SSH' BUFFER_SIZE = 512 def __init__(self, host, username, password, port=None, on_session_start=None, pkey=None, *args, **kwargs): ConnectionParams.__init__(self, host, port=port, on_session_start=on_session_start, pkey=pkey) - ExpectSession.__init__(self, *args, **kwargs) + AbstractSession.__init__(self, *args, **kwargs) if self.port is None: self.port = 22 @@ -94,7 +94,7 @@ def disconnect(self): # self._current_channel = None if self._handler: self._handler.close() - self._active = False + self.active = False def _send(self, command, logger): """Send message to device diff --git a/cloudshell/cli/session/tcp_session.py b/cloudshell/cli/session/tcp_session.py index 2e6f578..f86b21e 100644 --- a/cloudshell/cli/session/tcp_session.py +++ b/cloudshell/cli/session/tcp_session.py @@ -1,18 +1,18 @@ import socket from cloudshell.cli.session.connection_params import ConnectionParams -from cloudshell.cli.session.expect_session import ExpectSession +from cloudshell.cli.session.session import AbstractSession from cloudshell.cli.session.session_exceptions import SessionReadTimeout, SessionReadEmptyData -class TCPSession(ExpectSession, ConnectionParams): +class TCPSession(AbstractSession, ConnectionParams): SESSION_TYPE = 'TCP' BUFFER_SIZE = 1024 def __init__(self, host, port, on_session_start=None, *args, **kwargs): ConnectionParams.__init__(self, host=host, port=port, on_session_start=on_session_start) - ExpectSession.__init__(self, *args, **kwargs) + AbstractSession.__init__(self, *args, **kwargs) self._buffer_size = self.BUFFER_SIZE self._handler = None @@ -38,7 +38,7 @@ def disconnect(self): """ self._handler.close() - self._active = False + self.active = False def _send(self, command, logger): """Send message to the session diff --git a/cloudshell/cli/session/telnet_session.py b/cloudshell/cli/session/telnet_session.py index 73ae7fe..4aa1b21 100644 --- a/cloudshell/cli/session/telnet_session.py +++ b/cloudshell/cli/session/telnet_session.py @@ -4,7 +4,7 @@ from cloudshell.cli.service.action_map import Action from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.session.connection_params import ConnectionParams -from cloudshell.cli.session.expect_session import ExpectSession +from cloudshell.cli.session.session import AbstractSession from cloudshell.cli.session.session_exceptions import SessionException, SessionReadTimeout, SessionReadEmptyData @@ -12,14 +12,14 @@ class TelnetSessionException(SessionException): pass -class TelnetSession(ExpectSession, ConnectionParams): +class TelnetSession(AbstractSession, ConnectionParams): SESSION_TYPE = 'TELNET' AUTHENTICATION_ERROR_PATTERN = '%.*($|\n)' def __init__(self, host, username, password, port=None, on_session_start=None, *args, **kwargs): ConnectionParams.__init__(self, host, port=port, on_session_start=on_session_start) - ExpectSession.__init__(self, *args, **kwargs) + AbstractSession.__init__(self, *args, **kwargs) if hasattr(self, 'port') and self.port is None: self.port = 23 @@ -74,7 +74,7 @@ def disconnect(self): """ if self._handler: self._handler.close() - self._active = False + self.active = False def _send(self, command, logger): """send message / command to device diff --git a/cloudshell/cli/session/tl1_session.py b/cloudshell/cli/session/tl1_session.py index 385002e..7875409 100644 --- a/cloudshell/cli/session/tl1_session.py +++ b/cloudshell/cli/session/tl1_session.py @@ -53,7 +53,7 @@ def _connect_actions(self, prompt, logger): self.switch_name = '' if self.on_session_start and callable(self.on_session_start): self.on_session_start(self, logger) - self._active = True + self.active = True def hardware_expect(self, command, expected_string, logger, action_map=None, error_map=None, timeout=None, retries=None, check_action_loop_detector=True, empty_loop_timeout=None, diff --git a/tests/cli/session/test_expect_session.py b/tests/cli/session/test_abstract_session.py similarity index 79% rename from tests/cli/session/test_expect_session.py rename to tests/cli/session/test_abstract_session.py index 4d0bd7e..3814d01 100644 --- a/tests/cli/session/test_expect_session.py +++ b/tests/cli/session/test_abstract_session.py @@ -1,4 +1,3 @@ -from collections import OrderedDict from unittest import TestCase from unittest.mock import patch, Mock, call, MagicMock @@ -6,16 +5,16 @@ from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.service.error_map import Error from cloudshell.cli.service.error_map import ErrorMap -from cloudshell.cli.session.expect_session import ExpectSession, ActionLoopDetector +from cloudshell.cli.session.session import AbstractSession, ActionLoopDetector from cloudshell.cli.session.session_exceptions import SessionReadTimeout, ExpectedSessionException, \ SessionLoopLimitException, CommandExecutionException -class TestExpectSessionException(Exception): +class TestAbstractSessionException(Exception): pass -class ExpectSessionImpl(ExpectSession): +class AbstractSessionImpl(AbstractSession): def _initialize_session(self, prompt, logger): pass @@ -35,16 +34,16 @@ def _send(self, command, logger): pass -@patch("cloudshell.cli.session.expect_session.ActionLoopDetector.loops_detected", return_value=False) -@patch("cloudshell.cli.session.expect_session.normalize_buffer") -class TestExpectSession(TestCase): +@patch("cloudshell.cli.session.session.ActionLoopDetector.loops_detected", return_value=False) +@patch("cloudshell.cli.session.session.normalize_buffer") +class TestAbstractSession(TestCase): def setUp(self): self._logger = Mock() self._connect = Mock() self._disconnect = Mock() self._send = Mock() self._receive = Mock() - self._instance = ExpectSessionImpl() + self._instance = AbstractSessionImpl() self._instance._send = self._send self._instance._receive = self._receive self._instance.connect = self._connect @@ -60,7 +59,7 @@ def test_session_type(self, normalize_buffer, loops_detected): self.assertEqual(self._instance.session_type, 'EXPECT') def test_active(self, normalize_buffer, loops_detected): - self.assertFalse(self._instance.active()) + self.assertFalse(self._instance.active) def test_clear_buffer_receive_call(self, normalize_buffer, loops_detected): timeout = Mock() @@ -76,13 +75,13 @@ def test_clear_buffer_raise_exception(self, normalize_buffer, loops_detected): self._instance._clear_buffer(timeout, self._logger) def test_clear_buffer_exit_with_no_data(self, normalize_buffer, loops_detected): - self._receive.side_effect = ['', TestExpectSessionException('Breaking loop')] + self._receive.side_effect = ['', TestAbstractSessionException('Breaking loop')] timeout = Mock() self._instance._clear_buffer(timeout, self._logger) self.assertTrue(True) def test_clear_buffer_exit_on_second_attempt(self, normalize_buffer, loops_detected): - self._receive.side_effect = ['test', '', TestExpectSessionException('Breaking loop')] + self._receive.side_effect = ['test', '', TestAbstractSessionException('Breaking loop')] timeout = Mock() self._instance._clear_buffer(timeout, self._logger) mock_calls = [call(timeout, self._logger), call(timeout, self._logger)] @@ -112,9 +111,9 @@ def test_receive_get_all_data(self, normalize_buffer, loops_detected): result = self._instance._receive_all(2, self._logger) self.assertTrue(result and result == data1 + data2) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_clear_buffer_calls(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -126,9 +125,9 @@ def test_hardware_expect_clear_buffer_calls(self, clear_buffer, receive_all, sen call(self._instance._clear_buffer_timeout, self._logger)] clear_buffer.assert_has_calls(mock_calls) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_send_line_call(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -138,9 +137,9 @@ def test_hardware_expect_send_line_call(self, clear_buffer, receive_all, send_li self._instance.hardware_expect(command, expected_string, self._logger) send_line.assert_called_once_with(command, self._logger) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_empty_expected_string(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -151,9 +150,9 @@ def test_hardware_expect_empty_expected_string(self, clear_buffer, receive_all, with self.assertRaises(exception): self._instance.hardware_expect(command, expected_string, self._logger) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_raise_session_loop_limit_exceded(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -166,9 +165,9 @@ def test_hardware_expect_raise_session_loop_limit_exceded(self, clear_buffer, re with self.assertRaises(exception): self._instance.hardware_expect(command, expected_string, self._logger, retries=retries) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_receive_all_call(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -179,9 +178,9 @@ def test_hardware_expect_receive_all_call(self, clear_buffer, receive_all, send_ self._instance.hardware_expect(command, expected_string, self._logger, timeout=timeout) receive_all.assrrt_called_once_with(timeout, self._logger) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer") + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer") def test_hardware_expect_normalize_buffer_call(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -192,9 +191,9 @@ def test_hardware_expect_normalize_buffer_call(self, clear_buffer, receive_all, self._instance.hardware_expect(command, expected_string, self._logger, timeout=timeout) normalize_buffer.assrrt_called_once_with(expected_string) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer", return_value='') + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer", return_value='') def test_hardware_expect_remove_command_from_output(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -207,9 +206,9 @@ def test_hardware_expect_remove_command_from_output(self, clear_buffer, receive_ result = self._instance.hardware_expect(command, expected_string, self._logger, timeout=timeout) self.assertEqual(result.strip(), expected_string) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer", return_value='') + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer", return_value='') def test_hardware_expect_action_map_call(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -223,9 +222,9 @@ def test_hardware_expect_action_map_call(self, clear_buffer, receive_all, send_l self._instance.hardware_expect(command, expected_string, self._logger, action_map=action_map) test_func.assert_called_once_with(self._instance, self._logger) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer", return_value='') + @patch("cloudshell.cli.session.session.AbstractSession.send_line") + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer", return_value='') def test_hardware_expect_error_map_call(self, clear_buffer, receive_all, send_line, normalize_buffer, loops_detected): command = 'test_command' @@ -237,9 +236,9 @@ def test_hardware_expect_error_map_call(self, clear_buffer, receive_all, send_li with self.assertRaises(exception): result = self._instance.hardware_expect(command, expected_string, self._logger, error_map=error_map) - @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line", MagicMock()) - @patch("cloudshell.cli.session.expect_session.ExpectSession._receive_all") - @patch("cloudshell.cli.session.expect_session.ExpectSession._clear_buffer", + @patch("cloudshell.cli.session.session.AbstractSession.send_line", MagicMock()) + @patch("cloudshell.cli.session.session.AbstractSession._receive_all") + @patch("cloudshell.cli.session.session.AbstractSession._clear_buffer", MagicMock(return_value='')) def test_hardware_expect_error_map_call_with_exception(self, receive_all, normalize_buffer, loops_detected): diff --git a/tests/cli/session/test_ssh_session.py b/tests/cli/session/test_ssh_session.py index 4ca84cd..49c7453 100644 --- a/tests/cli/session/test_ssh_session.py +++ b/tests/cli/session/test_ssh_session.py @@ -379,7 +379,7 @@ def test_init_attributes(self): mandatory_attributes = ['username', '_handler', '_current_channel', 'password', '_buffer_size'] self.assertEqual(len(set(mandatory_attributes).difference(set(self._instance.__dict__.keys()))), 0) - @patch('cloudshell.cli.session.ssh_session.ExpectSession') + @patch('cloudshell.cli.session.ssh_session.AbstractSession') def test_eq(self, expect_session): self._instance = SSHSession(self._hostname, self._username, self._password, port=self._port, on_session_start=self._on_session_start) @@ -398,7 +398,7 @@ def test_eq(self, expect_session): self._instance.__eq__(SSHSession(self._hostname, self._username, '', port=self._port, on_session_start=self._on_session_start, pkey=pkey))) - @patch('cloudshell.cli.session.ssh_session.ExpectSession') + @patch('cloudshell.cli.session.ssh_session.AbstractSession') def test_eq_rsa(self, expect_session): pkey = paramiko.RSAKey.from_private_key(StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE) self._instance = SSHSession(self._hostname, self._username, self._password, port=self._port, diff --git a/tests/cli/session/test_telnet_session.py b/tests/cli/session/test_telnet_session.py index 03962a4..7e01af7 100644 --- a/tests/cli/session/test_telnet_session.py +++ b/tests/cli/session/test_telnet_session.py @@ -12,7 +12,7 @@ def setUp(self): self._port = 22 self._on_session_start = Mock() - @patch('cloudshell.cli.session.telnet_session.ExpectSession') + @patch('cloudshell.cli.session.session.AbstractSession') @patch('cloudshell.cli.session.telnet_session.ConnectionParams') def test_init_attributes(self, connection_params, expect_session): self._instance = TelnetSession(self._hostname, self._username, self._password, port=self._port, @@ -20,7 +20,7 @@ def test_init_attributes(self, connection_params, expect_session): mandatory_attributes = ['username', '_handler', 'password'] self.assertEqual(len(set(mandatory_attributes).difference(set(self._instance.__dict__.keys()))), 0) - @patch('cloudshell.cli.session.ssh_session.ExpectSession') + @patch('cloudshell.cli.session.ssh_session.AbstractSession') def test_eq(self, expect_session): self._instance = TelnetSession(self._hostname, self._username, self._password, port=self._port, on_session_start=self._on_session_start) diff --git a/tests/cli/test_session_pool_context_manager.py b/tests/cli/test_session_pool_context_manager.py index 1585cd6..0a41452 100644 --- a/tests/cli/test_session_pool_context_manager.py +++ b/tests/cli/test_session_pool_context_manager.py @@ -103,7 +103,7 @@ def test_exit_return_session_on_ignored_exception(self, command_mode_helper): def test_exit_remove_session_on_inactive(self, command_mode_helper): self._instance._initialize_cli_service = Mock() session_value = Mock() - session_value.active.return_value = False + session_value.active = False self._session_pool_manager.get_session.return_value = session_value with self._instance as session: pass From cd4b5d2ec5171f1a94c1298beeadb913595e23fa Mon Sep 17 00:00:00 2001 From: Anthony Date: Thu, 1 Aug 2019 16:31:57 +0300 Subject: [PATCH 21/59] move "hardware_expect" to separate class --- .../cli_demo/connect_to_ubuntu/runnur.py | 2 +- cloudshell/cli/service/cli_service_impl.py | 21 ++- cloudshell/cli/service/hardware_expect.py | 140 ++++++++++++++++++ cloudshell/cli/session/session.py | 16 +- 4 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 cloudshell/cli/service/hardware_expect.py diff --git a/cloudshell/cli/examples/cli_demo/connect_to_ubuntu/runnur.py b/cloudshell/cli/examples/cli_demo/connect_to_ubuntu/runnur.py index 4702e7e..a78a2e5 100644 --- a/cloudshell/cli/examples/cli_demo/connect_to_ubuntu/runnur.py +++ b/cloudshell/cli/examples/cli_demo/connect_to_ubuntu/runnur.py @@ -11,7 +11,7 @@ class CliCommandMode(CommandMode): ENTER_COMMAND = '' EXIT_COMMAND = 'exit' - def __init__(self,context): + def __init__(self, context): CommandMode.__init__(self, CliCommandMode.PROMPT, CliCommandMode.ENTER_COMMAND, CliCommandMode.EXIT_COMMAND) diff --git a/cloudshell/cli/service/cli_service_impl.py b/cloudshell/cli/service/cli_service_impl.py index 0726bc0..7b6c2ca 100644 --- a/cloudshell/cli/service/cli_service_impl.py +++ b/cloudshell/cli/service/cli_service_impl.py @@ -122,8 +122,25 @@ def send_command(self, command, expected_string=None, action_map=None, error_map if not logger: logger = self._logger self.session.logger = logger - output = self.session.hardware_expect(command, expected_string=expected_string, action_map=action_map, - error_map=error_map, logger=logger, *args, **kwargs) + + # option 1: + # one HardwareExpect instance per command + # output = HardwareExpect(session=session).hardware_expect(command=command) + + # option 2: + # HardwareExpect service without state that operates command ans session objects + # self.hardware_expect(command=command) + + # option 3: test + # output = CommandRunner(session=self.session).hardware_expect(command=command) + + # option 4: test + # output = CommandRunner(session=self.session, command=command, expected_string=expected_string, + # action_map=action_map, error_map=error_map, + # logger=logger, *args, **kwargs).hardware_expect() + + # output = self.session.hardware_expect(command, expected_string=expected_string, action_map=action_map, + # error_map=error_map, logger=logger, *args, **kwargs) if remove_prompt: output = re.sub(r'^.*{}.*$'.format(expected_string), '', output, flags=re.MULTILINE) return output diff --git a/cloudshell/cli/service/hardware_expect.py b/cloudshell/cli/service/hardware_expect.py new file mode 100644 index 0000000..0f105a5 --- /dev/null +++ b/cloudshell/cli/service/hardware_expect.py @@ -0,0 +1,140 @@ +import re +import time + +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap +from cloudshell.cli.service.action_map import ActionLoopDetector +from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer +from cloudshell.cli.session.session_exceptions import SessionLoopLimitException, ExpectedSessionException, \ + CommandExecutionException, SessionReadTimeout, SessionReadEmptyData + +# option 1: save state in Command object +# option 2: save state ib this class + + +class CommandRunner: + def __init__(self, session): + self._session = session + + def _send_command(self, command): + self.session._clear_buffer(self._clear_buffer_timeout, logger) + logger.debug(f'Command: {command}') + self.send_line(command, logger) + + def _remove_command_from_output(self, command, output): + """If option remove_command_from_output is set to True, look for command in output buffer, + + remove it in case of found + :param command: + :return: + """ + command_pattern = self._generate_command_pattern(command.command) + if re.search(command_pattern, output, flags=re.MULTILINE): + output = re.sub(command_pattern, '', output, count=1, flags=re.MULTILINE) + command.remove_command_from_output = False + + return output + + def _wait_for_response(self): + pass + + def hardware_expect(self, command, logger): + """Get response form the device and compare it to action_map, error_map and expected_string patterns, + + perform actions specified in action_map if any, and return output. + Raise Exception if receive empty response from device within a minute + """ + if command.command: + self._send_command(command.command) + + # Loop until one of the expressions is matched or MAX_RETRIES + # nothing is expected (usually used for exit) + output_list = list() + output_str = '' + retries_count = 0 + is_correct_exit = False + + while command.retries == 0 or retries_count < command.retries: + read_buffer = self._session._receive_all(command.timeout, logger) + + if read_buffer: + read_buffer = normalize_buffer(read_buffer) + logger.debug(read_buffer) + output_str += read_buffer + + if command.command and command.remove_command_from_output: + output_str = self._remove_command_from_output(output_str, command) + + retries_count = 0 + else: + retries_count += 1 + time.sleep(command.empty_loop_timeout) + continue + + if self.session.match_prompt(command.expected_string, output_str, logger): + # logger.debug('Expected str: {}'.format(expected_string)) + output_list.append(output_str) + is_correct_exit = True + + action_matched = command.action_map.process(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=command.check_action_loop_detector, + action_loop_detector=command.action_loop_detector) + + if action_matched: + output_list.append(output_str) + output_str = '' + + if is_correct_exit: + break + + if not is_correct_exit: + raise SessionLoopLimitException(f'Session Loop limit exceeded, {retries_count} loops') + + result_output = ''.join(output_list) + command.error_map.process(output=result_output, logger=logger) + + # Read buffer to the end. Useful when expected_string isn't last in buffer + result_output += self._clear_buffer(self._clear_buffer_timeout, logger) + return result_output + + +class Command: + MAX_LOOP_RETRIES = 20 + READ_TIMEOUT = 30 + EMPTY_LOOP_TIMEOUT = 0.5 + CLEAR_BUFFER_TIMEOUT = 0.1 + LOOP_DETECTOR_MAX_ACTION_LOOPS = 3 + LOOP_DETECTOR_MAX_COMBINATION_LENGTH = 4 + RECONNECT_TIMEOUT = 30 + + def __init__(self, command, expected_string, action_map=None, error_map=None, + timeout=None, retries=MAX_LOOP_RETRIES, check_action_loop_detector=True, empty_loop_timeout=None, + remove_command_from_output=True): + """ + + :param str command: command to send + :param str expected_string: expected string + :param collections.OrderedDict action_map: dict with {re_str: action} to trigger some action on received string + :param error_map: expected error map with subclass of CommandExecutionException or str + :type error_map: dict[str, CommandExecutionException|str] + :param int timeout: session timeout + :param int retries: maximal retries count + :param bool check_action_loop_detector: + :param bool empty_loop_timeout: + :param bool remove_command_from_output: In some switches the output string includes the command which was called. + The flag used to verify whether the the command string removed from the output string. + :rtype: str + """ + self.command = command + self.expected_string = expected_string + self.action_map = action_map or ActionMap() + self.error_map = error_map or ErrorMap() + + if check_action_loop_detector: + self.action_loop_detector = ActionLoopDetector( + max_loops=self._loop_detector_max_action_loops, + max_combination_length=self._loop_detector_max_combination_length) + else: + self.action_loop_detector = None diff --git a/cloudshell/cli/session/session.py b/cloudshell/cli/session/session.py index d0b9eed..c1b446d 100644 --- a/cloudshell/cli/session/session.py +++ b/cloudshell/cli/session/session.py @@ -12,13 +12,13 @@ class AbstractSession(metaclass=ABCMeta): SESSION_TYPE = 'EXPECT' - MAX_LOOP_RETRIES = 20 - READ_TIMEOUT = 30 - EMPTY_LOOP_TIMEOUT = 0.5 - CLEAR_BUFFER_TIMEOUT = 0.1 - LOOP_DETECTOR_MAX_ACTION_LOOPS = 3 - LOOP_DETECTOR_MAX_COMBINATION_LENGTH = 4 - RECONNECT_TIMEOUT = 30 + # MAX_LOOP_RETRIES = 20 + # READ_TIMEOUT = 30 + # EMPTY_LOOP_TIMEOUT = 0.5 + # CLEAR_BUFFER_TIMEOUT = 0.1 + # LOOP_DETECTOR_MAX_ACTION_LOOPS = 3 + # LOOP_DETECTOR_MAX_COMBINATION_LENGTH = 4 + # RECONNECT_TIMEOUT = 30 @abstractmethod def _send(self, command, logger): @@ -112,7 +112,7 @@ def _clear_buffer(self, timeout, logger): read_buffer = self._receive(timeout, logger) except (SessionReadTimeout, SessionReadEmptyData): read_buffer = None - if read_buffer: + if read_buffer: # todo: rework out += read_buffer else: break From 0073e4ba3ce89ced314206f49886e3946b43ba82 Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 11:39:30 +0200 Subject: [PATCH 22/59] Create py2-py3-packages-ci.yml --- .github/workflows/py2-py3-packages-ci.yml | 183 ++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 .github/workflows/py2-py3-packages-ci.yml diff --git a/.github/workflows/py2-py3-packages-ci.yml b/.github/workflows/py2-py3-packages-ci.yml new file mode 100644 index 0000000..fb403cf --- /dev/null +++ b/.github/workflows/py2-py3-packages-ci.yml @@ -0,0 +1,183 @@ +name: CI + +on: + push: + branches: + - "*" + pull_request: + branches: + - "*" + release: + types: [published] + +jobs: + tests: + name: Run unit tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.7] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox codecov + - name: Set TOXENV + run: | + python_version="${{ matrix.python-version }}" + py_version="${python_version/./}" + branch=(`[[ 'refs/head/master' == ${{ github.ref }} ]] && echo 'master' || echo 'dev'`) + TOXENV="py$py_version-$branch" + echo $TOXENV + echo "TOXENV=$TOXENV" >> $GITHUB_ENV + - name: Run tox + run: tox + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + file: .coverage + fail_ci_if_error: true + verbose: true + pre-commit: + name: Run pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox + - name: Run pre-commit + env: + TOXENV: pre-commit + run: tox + build: + name: Build package + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox + - name: Build + env: + TOXENV: build + run: tox + check-version: + name: Check version + # only for PRs in master + if: ${{ github.base_ref == 'master' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Check version + run: | + git clone https://github.com/${{ github.repository }}.git ${{ github.repository }} + cd ${{ github.repository }} + git checkout -qf ${{ github.head_ref }} + ! git diff --exit-code --quiet origin/master version.txt + deploy-to-test-pypi: + needs: [tests, pre-commit, build] + if: ${{ github.ref == 'refs/heads/dev' && github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox + - name: Add id to a package version + run: sed -i -E "s/^([0-9]+\.[0-9]+\.[0-9]+)$/\1.${{ github.run_number }}/" version.txt + - name: Build + env: + TOXENV: build + run: tox + - name: Publish + uses: pypa/gh-action-pypi-publish@v1.4.1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + create-gh-release: + needs: [tests, pre-commit, build] + if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox + - name: Build + env: + TOXENV: build + run: tox + - name: Set envs + run: | + version="$(cat version.txt | tr -d ' \t\n\r')" + repo_owner=${{ github.repository }} + index=`expr index "$repo_owner" /` + repo=${repo_owner:index} + echo "TAG=$version" >> $GITHUB_ENV + echo "REPO=$repo" >> $GITHUB_ENV + - name: Create GitHub release + uses: ncipollo/release-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + artifacts: "dist/*" + draft: true + name: ${{ env.REPO }} ${{ env.TAG }} + tag: ${{ env.TAG }} + commit: master + deploy-to-pypi: + needs: [tests, pre-commit, build] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install pip -U + pip install tox + - name: Build + env: + TOXENV: build + run: tox + - name: Publish + uses: pypa/gh-action-pypi-publish@v1.4.1 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} From a54a4936e0604ef420efca3abea0110410b14a47 Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 11:40:59 +0200 Subject: [PATCH 23/59] Delete .travis.yml --- .travis.yml | 89 ----------------------------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9b5ce21..0000000 --- a/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -language: python -python: 3.7 -jobs: - include: - - if: branch = master - python: 2.7 - env: TOXENV=py27-master - after_success: codecov - - if: branch = master - env: TOXENV=py37-master - after_success: codecov - - if: branch != master - python: 2.7 - env: TOXENV=py27-dev - after_success: codecov - - if: branch != master - env: TOXENV=py37-dev - after_success: codecov - - env: TOXENV=build - - env: TOXENV=pre-commit - - stage: Deploy to Test PyPI release - env: TOXENV=build - before_script: sed -i -E "s/^([0-9]+\.[0-9]+\.[0-9]+)$/\1.$TRAVIS_BUILD_NUMBER/" version.txt - deploy: - distributions: skip - skip_cleanup: true - provider: pypi - server: https://test.pypi.org/legacy/ - user: "__token__" - password: - secure: "MueXAWCLVldixZjac8QCbR2MW6l5fkXrUx3xrd4yWX3GN+SPMapnvO7KBhXOskGtgX9hEYeP1NkzkFiLcL4WvGYg+6BtKQO3MHgKEwgSoFq/Zb/cm9KP0kBhaMm06Xk+0B2HR7NjV/F2l4vlmaAw5jwYvbq4T8PATRXfeCYFcuy/HD6S2OldKvNMFDInpVrrgDki6jNK/Nxf1go5e225sdj/4xOU7ebRidDkiWGVIGSIdOh9ZZ2DN1uIhg+F14HwZAe+ebfYugBuNkBl4LqQi++rue36w5d7/W0cHlFeTHB2f26ncBPaExQJynCUn7aq+H/7JLRcv+ZmmK14rSQjk753n41749KmtVh0VK53D84oSOaeWYa3LgXYCqZJNO0QjS/PVKfvgrkU8BM77e1Pl4XYQxQt2BsxJqvzchbqzCno/UX2QJ+wpUGKUtwn3hoMKKJYYhI8e5YYJzzN1zcr85OkmrMzyAqNXYyAZnw6qneX+45N9Xs07j+WsnhZt+tTo72udpksV1MiOPY9K/wYSmiP1OttTohPxhaSNK3UOqyZ7BWrkugt+iovoDLWW+15D2Q5FdXw4pCnVlr/FqPJ27QfSC6t46JozqwTpY68e0FztfAkM7mJZiUyPN+g5mOq3/J3vP5OmYFNlpBDQdcNo/a7OlwTODhTaGej8dt97lg=" - on: - branch: dev - - stage: Deploy to PyPI release - env: TOXENV=build - deploy: - distributions: skip - skip_cleanup: true - provider: pypi - user: "__token__" - password: - secure: "dGuH2O4QDI+TKBGkxcWq9eSAAHHniaWPkW7OyFRf6786gBeiLTsHPxs1V0LhhKu2HLhniBqv11y0JEnbXpq32va+E5ImpDN03W+zGilq50cDiAzE8ihHRH8g4h7J4IJD/Hg6zriAkZY5CM7BwEFa3apuKbrnbsTXb8+ygSAlc+59lNjxzMd7rP4gC1fs5nkF/NLsGPH7zvcurs5JvT4inIFejiFDvqRa3H+6JMADI3Y+LCSDvD9eoH+dHrok38NURKAOcS6mfRAOBZEDhUpjG3DLm79G1BZNOv9O9fhzGNeNC49/TnLjLBAecMWlRqQhsCUEtypKoMtkEuJJoYgWn8dTPcW+FSo1a+NAnDi7jAB0ULqETzRD4vZNGH95rzJgi22mLms3Q3qQnfyr5PALXm42ikJpqXaGNhbWJBDWdTixHI+eJjuoGIWQXHoLmwsa3kVSKlTB/r2fsTfEQMYXky8ecZdUYiBBa9TqDX8cu2XDP/sn+VM4/kLa/wzsHe0gbO+m8XTcloPwYAy2UIjumLHxeJcyrDilwlmy9v2jLzN58juJZAwQrJyQMB6OZ5yAT5XFPKYgdM3AheWVRZytd30YUVsw9UlPzJPS5ZxyiDd94yZvOJzrp8VkKZV9M8bIkezad+zJRw59d/bP51Ot8Qx27JMXb9KeBZ7RD/obnRk=" - on: - tags: true - - stage: Create GitHub release - env: TOXENV=build - before_deploy: - - export AUTHOR_EMAIL="$(git log -1 $TRAVIS_COMMIT --pretty="%cE")" - - export AUTHOR_NAME="$(git log -1 $TRAVIS_COMMIT --pretty="%aN")" - - export GIT_TAG="$(cat version.txt | tr -d ' \t\n\r')" - - git config --local user.name $AUTHOR_NAME - - git config --local user.email $AUTHOR_EMAIL - - git tag $GIT_TAG - deploy: - provider: releases - skip_cleanup: true - draft: true - api_key: - secure: "Slv+t+YmXb8uA36XU1MQ6vwk+5Ta/XtmQZHyNHGqUg7hTVgqDZyx6dr+RspLZ3sVyO2T9VDb6R+V9Rl/HPUxkdAX/WjSgXH3X3M+ITNKA7KMSX9BOdWKDe+VtqWyUqdCpjfOyiOwr8HE/G/1qXlqNn8CZOb0MKM2VQ5AMPK3FnAnQEKoeFXcD9op+R5cLxeKH6ujkHPFrB0mJrag/b9kgZfP/WSVKNNqLZQaF1gw5s47wFAAau3p7ADc7ujtMaeeBmgbpKBESPPrRBzYSoZpSZ8dXcVeazcWwAIwjFHnbDOPXsa+xGbqBcCrknmFMAEJTakvSjr3eNLr4u67EQ3OeFpHBUuoZ+8MXPA9dRt76g9J0Ukch9NwBE4kaZKlp+tP1HlkB50FO9RCy7hgl6FQ1UFKwNUDIlBaACHX2tQZF6pvf8pkAduDECyOixw2txzBdhBxJYWlAMMOEudwElKvsRyt/3K07phy/xYSpqWvZN7hW7h3/Wd5PkaNfNQtGsb+ifT1KWbiV9uXKHLosjmUD0Tm8kWTIyerqOFzS3Q9bh7G0FSElLIRWMEeVKF/lQdXy+1Nd+DupG6nXrUz6HpJRm6FKWkbC3FF7WOQwRlTrvSVR5HJjjeXDYykjs+J5Bod3EqeyC4yr2axVwS0LEqyIjX0RFTk6gO/UEXmKvCv/vs=" - file_glob: true - file: dist/* - name: cloudshell-cli $GIT_TAG - target_commitish: master - on: - branch: master - - stage: Check version - language: bash - install: - - git clone https://github.com/$TRAVIS_REPO_SLUG.git $TRAVIS_REPO_SLUG - - cd $TRAVIS_REPO_SLUG - - git checkout -qf $TRAVIS_PULL_REQUEST_BRANCH - script: "! git diff --exit-code --quiet origin/master version.txt" - -install: - - pip install tox - - pip install codecov - -script: tox - -stages: - - name: Check version - if: branch = master AND type = pull_request - - name: Test - - name: Deploy to Test PyPI release - if: branch = dev AND type != pull_request - - name: Create GitHub release - if: branch = master AND type != pull_request - - name: Deploy to PyPI release - if: tag IS present From 5df9fbefbbc3fe32334fd2a06640f699410d723f Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 11:43:07 +0200 Subject: [PATCH 24/59] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6db26cf..b8b0ed4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # CloudShell CLI -[![Build status](https://travis-ci.org/QualiSystems/cloudshell-cli.svg?branch=dev)](https://travis-ci.org/QualiSystems/cloudshell-cli) +[![Build status](https://github.com/QualiSystems/cloudshell-cli/workflows/CI/badge.svg?branch=master)](https://github.com/QualiSystems/cloudshell-cli/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/QualiSystems/cloudshell-cli/branch/dev/graph/badge.svg)](https://codecov.io/gh/QualiSystems/cloudshell-cli) [![PyPI version](https://badge.fury.io/py/cloudshell-cli.svg)](https://badge.fury.io/py/cloudshell-cli) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) From 98909bf8f76419adcfec1bc404a366ca4eec4199 Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 12:01:58 +0200 Subject: [PATCH 25/59] update pre-commit, build the package quiet --- .pre-commit-config.yaml | 8 ++++---- .../examples/cli_demo/connect_to_AWS/runnur.py | 1 + .../cli_demo/connect_to_switch/runner.py | 3 ++- tox.ini | 16 +++++----------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7cc755..6a8bcd5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ repos: - - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 + - repo: https://github.com/timothycrosley/isort + rev: 5.6.4 hooks: - id: isort language_version: python3.7 - repo: https://github.com/python/black - rev: 19.3b0 + rev: 20.8b1 hooks: - id: black language_version: python3.7 - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: [ diff --git a/cloudshell/cli/examples/cli_demo/connect_to_AWS/runnur.py b/cloudshell/cli/examples/cli_demo/connect_to_AWS/runnur.py index 925cd71..fe298aa 100644 --- a/cloudshell/cli/examples/cli_demo/connect_to_AWS/runnur.py +++ b/cloudshell/cli/examples/cli_demo/connect_to_AWS/runnur.py @@ -1,6 +1,7 @@ import io import paramiko + from cloudshell.core.logger.qs_logger import get_qs_logger from cloudshell.cli.cli import CLI diff --git a/cloudshell/cli/examples/cli_demo/connect_to_switch/runner.py b/cloudshell/cli/examples/cli_demo/connect_to_switch/runner.py index 18a56d4..ae2fe70 100644 --- a/cloudshell/cli/examples/cli_demo/connect_to_switch/runner.py +++ b/cloudshell/cli/examples/cli_demo/connect_to_switch/runner.py @@ -1,9 +1,10 @@ +from connect_to_switch.SwitchClihandler import SwitchCliHandler + from cloudshell.core.logger.qs_logger import get_qs_logger from cloudshell.shell.standards.core import ( ResourceCommandContext, ResourceContextDetails, ) -from connect_to_switch.SwitchClihandler import SwitchCliHandler from cloudshell.cli.cli import CLI from cloudshell.cli.session_pool_manager import SessionPoolManager diff --git a/tox.ini b/tox.ini index 5305182..8440c90 100644 --- a/tox.ini +++ b/tox.ini @@ -2,9 +2,6 @@ # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. -[DEFAULT] -package-name = cloudshell.cli - [tox] envlist = py{27,37}-{master,dev} @@ -19,7 +16,7 @@ deps = master: -r test_requirements.txt dev: -r dev_requirements.txt commands = - nosetests --with-coverage --cover-package={[DEFAULT]package-name} tests + nosetests --with-coverage --cover-package=cloudshell.cli tests [testenv:pre-commit] basepython = python3 @@ -30,15 +27,12 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:build] skip_install = true commands = - python setup.py sdist --format zip - python setup.py bdist_wheel --universal + python setup.py -q sdist --format zip + python setup.py -q bdist_wheel --universal [isort] -line_length = 88 -forced_separate = %(package-name)s,tests -multi_line_output = 3 -include_trailing_comma = True -combine_as_imports = 1 +profile = black +forced_separate = cloudshell.cli,tests [flake8] max-line-length = 88 From ae7a8de1ee7859a93cd98170432ce28cbd606c2a Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 12:07:38 +0200 Subject: [PATCH 26/59] set cryptography < 3.3 from 3.3 cryptography creates wheels with abi3 tag that doesn't supported by old pip or offline repository --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 24eec04..c356b6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ bcrypt>=3.1.3,<3.2 +cryptography~=3.2.1 paramiko>=2.6,<2.7 scp>=0.13,<1 functools32; python_version <= '2.7' From dda48fad3ca6e3644fa549fb0ae88fcc7599c27f Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 12:08:07 +0200 Subject: [PATCH 27/59] update version --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index c4e41f9..c5106e6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.3 +4.0.4 From 839dc4489013deaa9a087ed68dc609e4d684675b Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Thu, 10 Dec 2020 12:27:03 +0200 Subject: [PATCH 28/59] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c356b6e..e928997 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bcrypt>=3.1.3,<3.2 -cryptography~=3.2.1 +cryptography>=2.5,<3.3 paramiko>=2.6,<2.7 scp>=0.13,<1 functools32; python_version <= '2.7' From c6b8a0dbc573a2a0073b5ab7c8619c4d0baf7088 Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Tue, 5 Jan 2021 14:43:01 +0200 Subject: [PATCH 29/59] fix getting the target branch --- .github/workflows/py2-py3-packages-ci.yml | 9 ++++++++- version.txt | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/py2-py3-packages-ci.yml b/.github/workflows/py2-py3-packages-ci.yml index fb403cf..8ae3323 100644 --- a/.github/workflows/py2-py3-packages-ci.yml +++ b/.github/workflows/py2-py3-packages-ci.yml @@ -32,7 +32,14 @@ jobs: run: | python_version="${{ matrix.python-version }}" py_version="${python_version/./}" - branch=(`[[ 'refs/head/master' == ${{ github.ref }} ]] && echo 'master' || echo 'dev'`) + target_branch=${{ github.base_ref || github.ref }} + target_branch=(`[[ ${target_branch::10} == 'refs/heads' ]] && echo ${target_branch:11} || echo $target_branch`) + echo "target_branch =" $target_branch + is_master=(`[[ $target_branch == 'master' ]] && echo 'true' || echo 'false'`) + is_tag=${{ startsWith(github.ref, 'refs/tags') }} + echo "is_master =" $is_master + echo "is_tag =" $is_tag + branch=(`[[ $is_master == 'true' || $is_tag == 'true' ]] && echo 'master' || echo 'dev'`) TOXENV="py$py_version-$branch" echo $TOXENV echo "TOXENV=$TOXENV" >> $GITHUB_ENV diff --git a/version.txt b/version.txt index c5106e6..7636e75 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.4 +4.0.5 From 1d25646629099981d6142e90bdf974b701774b78 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 14:45:32 +0300 Subject: [PATCH 30/59] Add basic implementation --- cloudshell/cli/configurator.py | 73 ++++++++++------------- cloudshell/cli/factory/__init__.py | 0 cloudshell/cli/factory/session_factory.py | 69 +++++++++++++++++++++ cloudshell/cli/session/ssh_session.py | 15 ++++- 4 files changed, 114 insertions(+), 43 deletions(-) create mode 100644 cloudshell/cli/factory/__init__.py create mode 100644 cloudshell/cli/factory/session_factory.py diff --git a/cloudshell/cli/configurator.py b/cloudshell/cli/configurator.py index d3d7ed5..54d0c96 100644 --- a/cloudshell/cli/configurator.py +++ b/cloudshell/cli/configurator.py @@ -2,7 +2,13 @@ # -*- coding: utf-8 -*- import sys from abc import ABCMeta, abstractmethod +from collections import defaultdict +from cloudshell.cli.factory.session_factory import ( + CloudInfoAccessKeySessionFactory, + GenericSessionFactory, + SessionFactory, +) from cloudshell.cli.service.cli import CLI from cloudshell.cli.session.ssh_session import SSHSession from cloudshell.cli.session.telnet_session import TelnetSession @@ -16,39 +22,30 @@ class CLIServiceConfigurator(object): - REGISTERED_SESSIONS = (SSHSession, TelnetSession) - - def __init__(self, resource_config, logger, cli=None, registered_sessions=None): + REGISTERED_SESSIONS = (CloudInfoAccessKeySessionFactory(SSHSession), TelnetSession) + """Using factories instead of """ + + def __init__( + self, + resource_config, + logger, + cli=None, + registered_sessions=None, + reservation_context=None, + ): """Initialize CLI service configurator. :param cloudshell.shell.standards.resource_config_generic_models.GenericCLIConfig resource_config: # noqa: E501 :param logging.Logger logger: :param cloudshell.cli.service.cli.CLI cli: :param registered_sessions: Session types and order + :param cloudshell.shell.core.driver_context.ReservationContextDetails reservation_context: """ self._cli = cli or CLI() self._resource_config = resource_config self._logger = logger self._registered_sessions = registered_sessions or self.REGISTERED_SESSIONS - - @property - def _username(self): - return self._resource_config.user - - @property - @lru_cache() - def _password(self): - return self._resource_config.password - - @property - def _resource_address(self): - """Resource IP.""" - return self._resource_config.address - - @property - def _port(self): - """Connection port property, to open socket on.""" - return self._resource_config.cli_tcp_port + self._reservation_context = reservation_context @property def _cli_type(self): @@ -58,29 +55,21 @@ def _cli_type(self): @property @lru_cache() def _session_dict(self): - return {sess.SESSION_TYPE.lower(): [sess] for sess in self._registered_sessions} - - def _on_session_start(self, session, logger): - """Perform some default commands when session just opened. - - Like 'no logging console' - """ - pass - - @property - @lru_cache() - def _session_kwargs(self): - return { - "host": self._resource_address, - "username": self._username, - "password": self._password, - "port": self._port, - "on_session_start": self._on_session_start, - } + session_dict = defaultdict(list) + for sess in self._registered_sessions: + session_dict[sess.SESSION_TYPE.lower()].append(sess) + return session_dict + + def initialize_session(self, session): + if not isinstance(session, SessionFactory): + session = GenericSessionFactory(session) + return session.init_session( + self._resource_config, self._logger, self._reservation_context + ) def _defined_sessions(self): return [ - sess(**self._session_kwargs) + self.initialize_session(sess) for sess in self._session_dict.get( self._cli_type.lower(), self._registered_sessions ) diff --git a/cloudshell/cli/factory/__init__.py b/cloudshell/cli/factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudshell/cli/factory/session_factory.py b/cloudshell/cli/factory/session_factory.py new file mode 100644 index 0000000..c3cb528 --- /dev/null +++ b/cloudshell/cli/factory/session_factory.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from abc import ABCMeta, abstractmethod + +ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) + + +class SessionFactory(ABC): + """Session factory. + + Help to initialize session for specified session class. + """ + + def __init__(self, session_class: type): + """:param session_class: Session class.""" + self.session_class = session_class + + @abstractmethod + def init_session(self, resource_config, logger, reservation_context=None): + """Initialize session instance. + + Encapsulate the logic of the session instance creation. + :param resource_config: + :param logging.Logger logger: + :param ReservationContextDetails reservation_context: + """ + raise NotImplementedError + + +class GenericSessionFactory(SessionFactory): + def init_session(self, resource_config, logger, reservation_context=None): + return self.session_class( + **self._session_kwargs(resource_config, logger, reservation_context) + ) + + @property + def SESSION_TYPE(self): + return self.session_class.SESSION_TYPE + + def _on_session_start(self, session, logger): + """Perform some default commands when session just opened. + + Like 'no logging console' + """ + pass + + def _session_kwargs(self, resource_config, logger, reservation_context=None): + return { + "host": resource_config.address, + "username": resource_config.user, + "password": resource_config.password, + "port": resource_config.cli_tcp_port, + "on_session_start": self._on_session_start, + } + + +class CloudInfoAccessKeySessionFactory(GenericSessionFactory): + def _session_kwargs(self, resource_config, logger, reservation_context=None): + access_key = "" + if reservation_context and reservation_context.cloud_info_access_key: + access_key = reservation_context.cloud_info_access_key + return { + "host": resource_config.address, + "username": resource_config.user, + "password": resource_config.password, + "port": resource_config.cli_tcp_port, + "pkey": access_key, + "on_session_start": self._on_session_start, + } diff --git a/cloudshell/cli/session/ssh_session.py b/cloudshell/cli/session/ssh_session.py index ee18650..37c5cf5 100644 --- a/cloudshell/cli/session/ssh_session.py +++ b/cloudshell/cli/session/ssh_session.py @@ -1,4 +1,5 @@ import socket +from io import StringIO import paramiko from scp import SCPClient @@ -86,7 +87,7 @@ def _initialize_session(self, prompt, logger): banner_timeout=30, allow_agent=False, look_for_keys=False, - pkey=self.pkey, + pkey=self._get_pkey_object(self.pkey, None, logger), ) except Exception as e: logger.exception("Failed to initialize session:") @@ -182,3 +183,15 @@ def upload_sftp( sftp.putfo(file_stream, dest_pathname) sftp.chmod(dest_pathname, int(dest_permissions, base=8)) sftp.close() + + @staticmethod + def _get_pkey_object(key_material, passphrase, logger): + """Try to detect private key type and return paramiko.PKey object.""" + for cls in [paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey]: + try: + key = cls.from_private_key(StringIO(key_material), password=passphrase) + except paramiko.ssh_exception.SSHException as e: + # Invalid key, try other key type + logger.warning(e) + else: + return key From 400391166daded25a5a0391cd0bbfc64f54f1176 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 16:50:56 +0300 Subject: [PATCH 31/59] Add phassphrase, fix unittests --- cloudshell/cli/session/ssh_session.py | 5 ++++- tests/cli/session/test_ssh_session.py | 25 ++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cloudshell/cli/session/ssh_session.py b/cloudshell/cli/session/ssh_session.py index 37c5cf5..1067b62 100644 --- a/cloudshell/cli/session/ssh_session.py +++ b/cloudshell/cli/session/ssh_session.py @@ -29,6 +29,7 @@ def __init__( port=None, on_session_start=None, pkey=None, + pkey_passphrase=None, *args, **kwargs ): @@ -43,6 +44,7 @@ def __init__( self.username = username self.password = password self.pkey = pkey + self.pkey_passphrase = pkey_passphrase self._handler = None self._current_channel = None @@ -59,6 +61,7 @@ def __eq__(self, other): self.username == other.username, self.password == other.password, self.pkey == other.pkey, + self.pkey_passphrase == other.pkey_passphrase, ] ) @@ -87,7 +90,7 @@ def _initialize_session(self, prompt, logger): banner_timeout=30, allow_agent=False, look_for_keys=False, - pkey=self._get_pkey_object(self.pkey, None, logger), + pkey=self._get_pkey_object(self.pkey, self.pkey_passphrase, logger), ) except Exception as e: logger.exception("Failed to initialize session:") diff --git a/tests/cli/session/test_ssh_session.py b/tests/cli/session/test_ssh_session.py index 6ec39db..f8c9381 100644 --- a/tests/cli/session/test_ssh_session.py +++ b/tests/cli/session/test_ssh_session.py @@ -443,9 +443,6 @@ def test_eq(self, expect_session): ) ) - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) self.assertFalse( self._instance.__eq__( SSHSession( @@ -454,23 +451,22 @@ def test_eq(self, expect_session): "", port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) ) ) @patch("cloudshell.cli.session.ssh_session.ExpectSession") def test_eq_rsa(self, expect_session): - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) self._instance = SSHSession( self._hostname, self._username, self._password, port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self.assertTrue( @@ -481,7 +477,8 @@ def test_eq_rsa(self, expect_session): self._password, port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) ) ) @@ -640,16 +637,13 @@ def test_rsa(self): "", port=server.port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self._instance.connect(">", logger=Mock()) self._instance.hardware_expect("dummy command", ">", Mock()) def test_rsa_failure(self): - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) - server = SSHServer(user2key={}) with self.assertRaises(SSHSessionException): @@ -659,7 +653,8 @@ def test_rsa_failure(self): "", port=server.port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self._instance.connect(">", logger=Mock()) self._instance.hardware_expect("dummy command", ">", Mock()) From 9a38ff37e91e7798511e860603f5a8a79b782472 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 16:52:59 +0300 Subject: [PATCH 32/59] Version 4.1.0 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 7636e75..ee74734 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.5 +4.1.0 From 684f28130a448320ebea91acfec50da02e3e3701 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Sun, 15 Nov 2020 17:36:35 +0200 Subject: [PATCH 33/59] take^Cndefined session list from session dict --- cloudshell/cli/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudshell/cli/configurator.py b/cloudshell/cli/configurator.py index d3d7ed5..0b30bca 100644 --- a/cloudshell/cli/configurator.py +++ b/cloudshell/cli/configurator.py @@ -82,7 +82,7 @@ def _defined_sessions(self): return [ sess(**self._session_kwargs) for sess in self._session_dict.get( - self._cli_type.lower(), self._registered_sessions + self._cli_type.lower(), self._session_dict.values() ) ] From 8645b2e2f17b892cc01ce086e1fb37fcc1985319 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Sun, 15 Nov 2020 17:59:33 +0200 Subject: [PATCH 34/59] Create list from list of lists --- cloudshell/cli/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudshell/cli/configurator.py b/cloudshell/cli/configurator.py index 0b30bca..931bc99 100644 --- a/cloudshell/cli/configurator.py +++ b/cloudshell/cli/configurator.py @@ -82,7 +82,7 @@ def _defined_sessions(self): return [ sess(**self._session_kwargs) for sess in self._session_dict.get( - self._cli_type.lower(), self._session_dict.values() + self._cli_type.lower(), sum(self._session_dict.values(), []) ) ] From 7a00648f8d22ae12eba860cc1e4e1ec2e7ce97b8 Mon Sep 17 00:00:00 2001 From: Kyrylo Maksymenko Date: Tue, 5 Jan 2021 14:43:01 +0200 Subject: [PATCH 35/59] fix getting the target branch --- .github/workflows/py2-py3-packages-ci.yml | 9 ++++++++- version.txt | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/py2-py3-packages-ci.yml b/.github/workflows/py2-py3-packages-ci.yml index fb403cf..8ae3323 100644 --- a/.github/workflows/py2-py3-packages-ci.yml +++ b/.github/workflows/py2-py3-packages-ci.yml @@ -32,7 +32,14 @@ jobs: run: | python_version="${{ matrix.python-version }}" py_version="${python_version/./}" - branch=(`[[ 'refs/head/master' == ${{ github.ref }} ]] && echo 'master' || echo 'dev'`) + target_branch=${{ github.base_ref || github.ref }} + target_branch=(`[[ ${target_branch::10} == 'refs/heads' ]] && echo ${target_branch:11} || echo $target_branch`) + echo "target_branch =" $target_branch + is_master=(`[[ $target_branch == 'master' ]] && echo 'true' || echo 'false'`) + is_tag=${{ startsWith(github.ref, 'refs/tags') }} + echo "is_master =" $is_master + echo "is_tag =" $is_tag + branch=(`[[ $is_master == 'true' || $is_tag == 'true' ]] && echo 'master' || echo 'dev'`) TOXENV="py$py_version-$branch" echo $TOXENV echo "TOXENV=$TOXENV" >> $GITHUB_ENV diff --git a/version.txt b/version.txt index c5106e6..7636e75 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.4 +4.0.5 From e2e7a5931543cc92f2b672c10b70f7cd3373c114 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 14:45:32 +0300 Subject: [PATCH 36/59] Add basic implementation --- cloudshell/cli/configurator.py | 73 ++++++++++------------- cloudshell/cli/factory/__init__.py | 0 cloudshell/cli/factory/session_factory.py | 69 +++++++++++++++++++++ cloudshell/cli/session/ssh_session.py | 15 ++++- 4 files changed, 114 insertions(+), 43 deletions(-) create mode 100644 cloudshell/cli/factory/__init__.py create mode 100644 cloudshell/cli/factory/session_factory.py diff --git a/cloudshell/cli/configurator.py b/cloudshell/cli/configurator.py index 931bc99..277d562 100644 --- a/cloudshell/cli/configurator.py +++ b/cloudshell/cli/configurator.py @@ -2,7 +2,13 @@ # -*- coding: utf-8 -*- import sys from abc import ABCMeta, abstractmethod +from collections import defaultdict +from cloudshell.cli.factory.session_factory import ( + CloudInfoAccessKeySessionFactory, + GenericSessionFactory, + SessionFactory, +) from cloudshell.cli.service.cli import CLI from cloudshell.cli.session.ssh_session import SSHSession from cloudshell.cli.session.telnet_session import TelnetSession @@ -16,39 +22,30 @@ class CLIServiceConfigurator(object): - REGISTERED_SESSIONS = (SSHSession, TelnetSession) - - def __init__(self, resource_config, logger, cli=None, registered_sessions=None): + REGISTERED_SESSIONS = (CloudInfoAccessKeySessionFactory(SSHSession), TelnetSession) + """Using factories instead of """ + + def __init__( + self, + resource_config, + logger, + cli=None, + registered_sessions=None, + reservation_context=None, + ): """Initialize CLI service configurator. :param cloudshell.shell.standards.resource_config_generic_models.GenericCLIConfig resource_config: # noqa: E501 :param logging.Logger logger: :param cloudshell.cli.service.cli.CLI cli: :param registered_sessions: Session types and order + :param cloudshell.shell.core.driver_context.ReservationContextDetails reservation_context: """ self._cli = cli or CLI() self._resource_config = resource_config self._logger = logger self._registered_sessions = registered_sessions or self.REGISTERED_SESSIONS - - @property - def _username(self): - return self._resource_config.user - - @property - @lru_cache() - def _password(self): - return self._resource_config.password - - @property - def _resource_address(self): - """Resource IP.""" - return self._resource_config.address - - @property - def _port(self): - """Connection port property, to open socket on.""" - return self._resource_config.cli_tcp_port + self._reservation_context = reservation_context @property def _cli_type(self): @@ -58,29 +55,21 @@ def _cli_type(self): @property @lru_cache() def _session_dict(self): - return {sess.SESSION_TYPE.lower(): [sess] for sess in self._registered_sessions} - - def _on_session_start(self, session, logger): - """Perform some default commands when session just opened. - - Like 'no logging console' - """ - pass - - @property - @lru_cache() - def _session_kwargs(self): - return { - "host": self._resource_address, - "username": self._username, - "password": self._password, - "port": self._port, - "on_session_start": self._on_session_start, - } + session_dict = defaultdict(list) + for sess in self._registered_sessions: + session_dict[sess.SESSION_TYPE.lower()].append(sess) + return session_dict + + def initialize_session(self, session): + if not isinstance(session, SessionFactory): + session = GenericSessionFactory(session) + return session.init_session( + self._resource_config, self._logger, self._reservation_context + ) def _defined_sessions(self): return [ - sess(**self._session_kwargs) + self.initialize_session(sess) for sess in self._session_dict.get( self._cli_type.lower(), sum(self._session_dict.values(), []) ) diff --git a/cloudshell/cli/factory/__init__.py b/cloudshell/cli/factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudshell/cli/factory/session_factory.py b/cloudshell/cli/factory/session_factory.py new file mode 100644 index 0000000..c3cb528 --- /dev/null +++ b/cloudshell/cli/factory/session_factory.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from abc import ABCMeta, abstractmethod + +ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) + + +class SessionFactory(ABC): + """Session factory. + + Help to initialize session for specified session class. + """ + + def __init__(self, session_class: type): + """:param session_class: Session class.""" + self.session_class = session_class + + @abstractmethod + def init_session(self, resource_config, logger, reservation_context=None): + """Initialize session instance. + + Encapsulate the logic of the session instance creation. + :param resource_config: + :param logging.Logger logger: + :param ReservationContextDetails reservation_context: + """ + raise NotImplementedError + + +class GenericSessionFactory(SessionFactory): + def init_session(self, resource_config, logger, reservation_context=None): + return self.session_class( + **self._session_kwargs(resource_config, logger, reservation_context) + ) + + @property + def SESSION_TYPE(self): + return self.session_class.SESSION_TYPE + + def _on_session_start(self, session, logger): + """Perform some default commands when session just opened. + + Like 'no logging console' + """ + pass + + def _session_kwargs(self, resource_config, logger, reservation_context=None): + return { + "host": resource_config.address, + "username": resource_config.user, + "password": resource_config.password, + "port": resource_config.cli_tcp_port, + "on_session_start": self._on_session_start, + } + + +class CloudInfoAccessKeySessionFactory(GenericSessionFactory): + def _session_kwargs(self, resource_config, logger, reservation_context=None): + access_key = "" + if reservation_context and reservation_context.cloud_info_access_key: + access_key = reservation_context.cloud_info_access_key + return { + "host": resource_config.address, + "username": resource_config.user, + "password": resource_config.password, + "port": resource_config.cli_tcp_port, + "pkey": access_key, + "on_session_start": self._on_session_start, + } diff --git a/cloudshell/cli/session/ssh_session.py b/cloudshell/cli/session/ssh_session.py index ee18650..37c5cf5 100644 --- a/cloudshell/cli/session/ssh_session.py +++ b/cloudshell/cli/session/ssh_session.py @@ -1,4 +1,5 @@ import socket +from io import StringIO import paramiko from scp import SCPClient @@ -86,7 +87,7 @@ def _initialize_session(self, prompt, logger): banner_timeout=30, allow_agent=False, look_for_keys=False, - pkey=self.pkey, + pkey=self._get_pkey_object(self.pkey, None, logger), ) except Exception as e: logger.exception("Failed to initialize session:") @@ -182,3 +183,15 @@ def upload_sftp( sftp.putfo(file_stream, dest_pathname) sftp.chmod(dest_pathname, int(dest_permissions, base=8)) sftp.close() + + @staticmethod + def _get_pkey_object(key_material, passphrase, logger): + """Try to detect private key type and return paramiko.PKey object.""" + for cls in [paramiko.RSAKey, paramiko.DSSKey, paramiko.ECDSAKey]: + try: + key = cls.from_private_key(StringIO(key_material), password=passphrase) + except paramiko.ssh_exception.SSHException as e: + # Invalid key, try other key type + logger.warning(e) + else: + return key From 13b8c5d351e7afe489c07affde8f0d9edce4836e Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 16:50:56 +0300 Subject: [PATCH 37/59] Add phassphrase, fix unittests --- cloudshell/cli/session/ssh_session.py | 5 ++++- tests/cli/session/test_ssh_session.py | 25 ++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cloudshell/cli/session/ssh_session.py b/cloudshell/cli/session/ssh_session.py index 37c5cf5..1067b62 100644 --- a/cloudshell/cli/session/ssh_session.py +++ b/cloudshell/cli/session/ssh_session.py @@ -29,6 +29,7 @@ def __init__( port=None, on_session_start=None, pkey=None, + pkey_passphrase=None, *args, **kwargs ): @@ -43,6 +44,7 @@ def __init__( self.username = username self.password = password self.pkey = pkey + self.pkey_passphrase = pkey_passphrase self._handler = None self._current_channel = None @@ -59,6 +61,7 @@ def __eq__(self, other): self.username == other.username, self.password == other.password, self.pkey == other.pkey, + self.pkey_passphrase == other.pkey_passphrase, ] ) @@ -87,7 +90,7 @@ def _initialize_session(self, prompt, logger): banner_timeout=30, allow_agent=False, look_for_keys=False, - pkey=self._get_pkey_object(self.pkey, None, logger), + pkey=self._get_pkey_object(self.pkey, self.pkey_passphrase, logger), ) except Exception as e: logger.exception("Failed to initialize session:") diff --git a/tests/cli/session/test_ssh_session.py b/tests/cli/session/test_ssh_session.py index 6ec39db..f8c9381 100644 --- a/tests/cli/session/test_ssh_session.py +++ b/tests/cli/session/test_ssh_session.py @@ -443,9 +443,6 @@ def test_eq(self, expect_session): ) ) - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) self.assertFalse( self._instance.__eq__( SSHSession( @@ -454,23 +451,22 @@ def test_eq(self, expect_session): "", port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) ) ) @patch("cloudshell.cli.session.ssh_session.ExpectSession") def test_eq_rsa(self, expect_session): - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) self._instance = SSHSession( self._hostname, self._username, self._password, port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self.assertTrue( @@ -481,7 +477,8 @@ def test_eq_rsa(self, expect_session): self._password, port=self._port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) ) ) @@ -640,16 +637,13 @@ def test_rsa(self): "", port=server.port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self._instance.connect(">", logger=Mock()) self._instance.hardware_expect("dummy command", ">", Mock()) def test_rsa_failure(self): - pkey = paramiko.RSAKey.from_private_key( - StringIO(KEY_WITH_PASSPHRASE), password=KEY_PASSPHRASE - ) - server = SSHServer(user2key={}) with self.assertRaises(SSHSessionException): @@ -659,7 +653,8 @@ def test_rsa_failure(self): "", port=server.port, on_session_start=self._on_session_start, - pkey=pkey, + pkey=KEY_WITH_PASSPHRASE, + pkey_passphrase=KEY_PASSPHRASE, ) self._instance.connect(">", logger=Mock()) self._instance.hardware_expect("dummy command", ">", Mock()) From e13db0fb407c88f95ba0a1897e74f6ea1170b1af Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 19 Jul 2021 16:52:59 +0300 Subject: [PATCH 38/59] Version 4.1.0 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 7636e75..ee74734 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.0.5 +4.1.0 From 48f88dcc320c42914f43d3dde20173b47b40a392 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Thu, 12 Aug 2021 17:04:12 +0300 Subject: [PATCH 39/59] Version 5.0.0 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index ee74734..0062ac9 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -4.1.0 +5.0.0 From 89accbe8a42f5af1a57295974ffc36bf11263c65 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Fri, 29 May 2020 14:40:46 +0300 Subject: [PATCH 40/59] Added cli_service_helper, implemented send command with retries --- cloudshell/cli/service/cli_service_helper.py | 66 ++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 cloudshell/cli/service/cli_service_helper.py diff --git a/cloudshell/cli/service/cli_service_helper.py b/cloudshell/cli/service/cli_service_helper.py new file mode 100644 index 0000000..8e24506 --- /dev/null +++ b/cloudshell/cli/service/cli_service_helper.py @@ -0,0 +1,66 @@ +class SendCommandWithRetries(object): + """ + Help to execute command with retries. + + If command execution raise an exception it helps + to reconnect or create a new session from the list of + defined sessions, optional. + """ + + MAX_RECREATE_RETRIES = 3 + MAX_RECONNECT_RETRIES = 0 + RECONNECT_TIMEOUT = 30 + + def __init__( + self, + cli_configurator, + command_mode, + logger, + recreate_retries=None, + reconnect_retries=None, + reconnect_timeout=None, + ): + """ + Init method. + + :param cloudshell.cli.configurator.CLIServiceConfigurator cli_configurator: + :param cloudshell.cli.service.command_mode.CommandMode command_mode: + :param logging.Logger logger: + :param int recreate_retries: + :param int reconnect_retries: + :param int reconnect_timeout: + """ + self.cli_configurator = cli_configurator + self.command_mode = command_mode + self._logger = logger + self._recreate_retries = recreate_retries or self.MAX_RECREATE_RETRIES + self._reconnect_retries = reconnect_retries or self.MAX_RECONNECT_RETRIES + self._reconnect_timeout = reconnect_timeout or self.RECONNECT_TIMEOUT + + def _send_command_with_reconnect(self, cli_service, *args, **kwargs): + """Send command with reconnect retries.""" + retry = 0 + while True: + try: + return cli_service.send_command(*args, **kwargs) + except Exception: + self._logger.exception("Reconnect retry {}".format(retry)) + if retry < self._reconnect_retries: + cli_service.reconnect(self._reconnect_timeout) + retry += 1 + else: + raise + + def send_command(self, *args, **kwargs): + """Send command with retries on fail.""" + retry = 0 + while retry < self._recreate_retries: + try: + with self.cli_configurator.get_cli_service( + self.command_mode + ) as cli_service: + self._send_command_with_reconnect(cli_service, *args, **kwargs) + except Exception: + self._logger.exception("Recreate retry {}".format(retry)) + retry += 1 + raise Exception("Max retries exceeded") From 948b8d4ae2943e1a3f8991c9baaf375850e55ad2 Mon Sep 17 00:00:00 2001 From: Yaroslav Nikonorov Date: Mon, 15 Jun 2020 14:05:26 +0300 Subject: [PATCH 41/59] Fixed default values. Fixed added return --- ...rvice_helper.py => cli_service_helpers.py} | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) rename cloudshell/cli/service/{cli_service_helper.py => cli_service_helpers.py} (75%) diff --git a/cloudshell/cli/service/cli_service_helper.py b/cloudshell/cli/service/cli_service_helpers.py similarity index 75% rename from cloudshell/cli/service/cli_service_helper.py rename to cloudshell/cli/service/cli_service_helpers.py index 8e24506..80aaa0a 100644 --- a/cloudshell/cli/service/cli_service_helper.py +++ b/cloudshell/cli/service/cli_service_helpers.py @@ -16,9 +16,9 @@ def __init__( cli_configurator, command_mode, logger, - recreate_retries=None, - reconnect_retries=None, - reconnect_timeout=None, + recreate_retries=MAX_RECREATE_RETRIES, + reconnect_retries=MAX_RECONNECT_RETRIES, + reconnect_timeout=RECONNECT_TIMEOUT, ): """ Init method. @@ -33,12 +33,17 @@ def __init__( self.cli_configurator = cli_configurator self.command_mode = command_mode self._logger = logger - self._recreate_retries = recreate_retries or self.MAX_RECREATE_RETRIES - self._reconnect_retries = reconnect_retries or self.MAX_RECONNECT_RETRIES - self._reconnect_timeout = reconnect_timeout or self.RECONNECT_TIMEOUT + self._recreate_retries = recreate_retries + self._reconnect_retries = reconnect_retries + self._reconnect_timeout = reconnect_timeout def _send_command_with_reconnect(self, cli_service, *args, **kwargs): - """Send command with reconnect retries.""" + """Send command with reconnect retries. + + :param cloudshell.cli.service.cli_service.CliService cli_service: + :param args: + :param kwargs: + """ retry = 0 while True: try: @@ -59,7 +64,9 @@ def send_command(self, *args, **kwargs): with self.cli_configurator.get_cli_service( self.command_mode ) as cli_service: - self._send_command_with_reconnect(cli_service, *args, **kwargs) + return self._send_command_with_reconnect( + cli_service, *args, **kwargs + ) except Exception: self._logger.exception("Recreate retry {}".format(retry)) retry += 1 From fed7cfd6bca8c3384502ba045f17d70f697d0ebe Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 24 Jun 2019 17:47:31 +0300 Subject: [PATCH 42/59] add "Action" and "ActionMap" classes --- cloudshell/cli/service/action_map.py | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 cloudshell/cli/service/action_map.py diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py new file mode 100644 index 0000000..e54dc9e --- /dev/null +++ b/cloudshell/cli/service/action_map.py @@ -0,0 +1,144 @@ +from collections import OrderedDict +import re + + +class Action(object): + def __init__(self, pattern, callback, execute_once=False): + """ + + :param str pattern: + :param function callback: + :param bool execute_once: + """ + self.pattern = pattern + self.callback = callback + self.execute_once = execute_once + + def __call__(self, session, logger): + """ + + :param cloudshell.cli.session.expect_session.ExpectSession session: + :param logging.Logger logger: + :return: + """ + return self.callback(session, logger) + + def __repr__(self): + """ + + :rtype: str + """ + return "{} pattern: {}, execute once: {}".format(super(Action, self).__repr__(), + self.pattern, + self.execute_once) + + def match(self, output): + """ + + :param str output: + :rtype: bool + """ + return bool(re.search(self.pattern, output, re.DOTALL)) + + +class ActionMap(object): + def __init__(self, actions=None): + """ + + :param list[Action] actions: + """ + if actions is None: + actions = [] + + self.matched_patterns = set() + self._actions_dict = OrderedDict([(action.pattern, action) for action in actions]) + + @property + def actions(self): + """ + + :rtype: list[Action] + """ + return [action for action in self._actions_dict.values()] + + @property + def active_actions(self): + """ + + :rtype: list[Action] + """ + return [action for action in self.actions if (not action.execute_once or + action.pattern not in self.matched_patterns)] + + def add(self, action): + """ + + :param Action action: + :return: + """ + self._actions_dict[action.pattern] = action + + def extend(self, action_map, override=False): + """ + + :param ActionMap action_map: + :param bool override: + :return: + """ + for action in action_map.actions: + if not override and action.pattern in self._actions_dict: + continue + self.add(action) + + self.matched_patterns |= action_map.matched_patterns + + def __call__(self, session, logger, output): + """ + + :param cloudshell.cli.session.expect_session.ExpectSession session: + :param logging.Logger logger: + :param str output: + :return: + """ + for action in self.active_actions: + if action.match(output): + self.matched_patterns.add(action.pattern) + return action(session, logger) + + def __add__(self, other): + """ + + :param other: + :rtype: ActionMap + """ + if isinstance(other, type(self)): + return ActionMap(actions=self.actions + other.actions) + + raise TypeError("unsupported operand type(s) for +: '{}' and '{}'".format(type(self), type(other))) + + def __repr__(self): + """ + + :rtype: str + """ + return "{} matched patterns: {}, actions: {}".format(super(ActionMap, self).__repr__(), + self.matched_patterns, + self.actions) + + +# if __name__ == "__main__": +# action_map1 = ActionMap(actions=[Action(pattern='action1', callback=lambda session, logger: session), +# Action(pattern='action2', callback=lambda session, logger: session)]) +# +# action_map1.add(Action(pattern="action3", +# callback=lambda session, logger: session, +# execute_once=True)) +# +# action_map2 = ActionMap(actions=[Action(pattern='action10', callback=lambda session, logger: session), +# Action(pattern='action20', callback=lambda session, logger: session)]) +# +# action_map2.add(Action(pattern="action1", +# callback=lambda session, logger: session, +# execute_once=True)) +# +# action_map2.extend(action_map1) From 3312f85b331b6f5626b221935364d71307146525 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jun 2019 16:27:24 +0300 Subject: [PATCH 43/59] update ActionMap usage in the "hardware_expect" method --- cloudshell/cli/service/action_map.py | 86 ++++++++++++++---- cloudshell/cli/session/expect_session.py | 109 ++++------------------- 2 files changed, 84 insertions(+), 111 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index e54dc9e..09c644d 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -1,6 +1,8 @@ from collections import OrderedDict import re +from cloudshell.cli.session.session_exceptions import SessionLoopDetectorException + class Action(object): def __init__(self, pattern, callback, execute_once=False): @@ -92,18 +94,32 @@ def extend(self, action_map, override=False): self.matched_patterns |= action_map.matched_patterns - def __call__(self, session, logger, output): + def __call__(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ :param cloudshell.cli.session.expect_session.ExpectSession session: :param logging.Logger logger: :param str output: - :return: + :param bool check_action_loop_detector: + :param ActionLoopDetector action_loop_detector: + :rtype: bool """ for action in self.active_actions: if action.match(output): + logger.debug("Matched Action with pattern: {}".format(action.pattern)) + + if check_action_loop_detector: + logger.debug("Checking loops fro Action with pattern : {}".format(action.pattern)) + + if action_loop_detector.loops_detected(action.pattern): + logger.error("Loops detected for action patter: {}".format(action.pattern)) + raise SessionLoopDetectorException("Expected actions loops detected") + + action(session, logger) self.matched_patterns.add(action.pattern) - return action(session, logger) + return True + + return False def __add__(self, other): """ @@ -126,19 +142,51 @@ def __repr__(self): self.actions) -# if __name__ == "__main__": -# action_map1 = ActionMap(actions=[Action(pattern='action1', callback=lambda session, logger: session), -# Action(pattern='action2', callback=lambda session, logger: session)]) -# -# action_map1.add(Action(pattern="action3", -# callback=lambda session, logger: session, -# execute_once=True)) -# -# action_map2 = ActionMap(actions=[Action(pattern='action10', callback=lambda session, logger: session), -# Action(pattern='action20', callback=lambda session, logger: session)]) -# -# action_map2.add(Action(pattern="action1", -# callback=lambda session, logger: session, -# execute_once=True)) -# -# action_map2.extend(action_map1) +class ActionLoopDetector(object): + """Help to detect loops for action combinations""" + + def __init__(self, max_loops, max_combination_length): + """ + + :param max_loops: + :param max_combination_length: + :return: + """ + self._max_action_loops = max_loops + self._max_combination_length = max_combination_length + self._action_history = [] + + def loops_detected(self, action_pattern): + """Add action key to the history and detect loops + + :param str action_pattern: + :return: + """ + self._action_history.append(action_pattern) + for combination_length in range(1, self._max_combination_length + 1): + if self._is_combination_compatible(combination_length): + if self._is_loop_exists(combination_length): + return True + return False + + def _is_combination_compatible(self, combination_length): + """Check if combinations may exist + + :param combination_length: + :return: + """ + return len(self._action_history) / combination_length >= self._max_action_loops + + def _is_loop_exists(self, combination_length): + """Detect loops for combination length + + :param combination_length: + :return: + """ + reversed_history = self._action_history[::-1] + combinations = [reversed_history[x:x + combination_length] for x in + range(0, len(reversed_history), combination_length)][:self._max_action_loops] + for x, y in [combinations[x:x + 2] for x in range(0, len(combinations) - 1)]: + if x != y: + return False + return True diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 446cc04..1e50723 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,18 +1,13 @@ import re import time from abc import ABCMeta, abstractmethod -from collections import OrderedDict +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.action_map import ActionLoopDetector from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer from cloudshell.cli.session.session import Session -from cloudshell.cli.session.session_exceptions import ( - CommandExecutionException, - ExpectedSessionException, - SessionLoopDetectorException, - SessionLoopLimitException, - SessionReadEmptyData, - SessionReadTimeout, -) +from cloudshell.cli.session.session_exceptions import SessionLoopLimitException, \ + ExpectedSessionException, CommandExecutionException, SessionReadTimeout, SessionReadEmptyData ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) @@ -217,10 +212,8 @@ def hardware_expect( :param command: command to send :param expected_string: expected string :param logger: logger - :param action_map: dict with {re_str: action} to trigger some action - on received string - :param error_map: expected error map with subclass of CommandExecutionException - or str + :param action_map: dict with {re_str: action} to trigger some action on received string + :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, CommandExecutionException|str] :param timeout: session timeout :param retries: maximal retries count @@ -231,10 +224,10 @@ def hardware_expect( :rtype: str """ if not action_map: - action_map = OrderedDict() + action_map = ActionMap() if not error_map: - error_map = OrderedDict() + error_map = ActionMap() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout @@ -288,22 +281,15 @@ def hardware_expect( output_list.append(output_str) is_correct_exit = True - for action_key in action_map: - result_match = re.search(action_key, output_str, re.DOTALL) - if result_match: - output_list.append(output_str) - - if check_action_loop_detector: - if action_loop_detector.loops_detected(action_key): - logger.error("Loops detected") - raise SessionLoopDetectorException( - self.__class__.__name__, - "Expected actions loops detected", - ) - logger.debug("Action key: {}".format(action_key)) - action_map[action_key](self, logger) - output_str = "" - break + action_matched = action_map(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=check_action_loop_detector, + action_loop_detector=action_loop_detector) + + if action_matched: + output_list.append(output_str) + output_str = '' if is_correct_exit: break @@ -353,64 +339,3 @@ def reconnect(self, prompt, logger, timeout=None): self.__class__.__name__, "Reconnect unsuccessful, timeout exceeded, see logs for more details", ) - - -class ActionLoopDetector(object): - """Help to detect loops for action combinations.""" - - def __init__(self, max_loops, max_combination_length): - """Help to detect loops for action combinations. - - :param max_loops: - :param max_combination_length: - :return: - """ - self._max_action_loops = max_loops - self._max_combination_length = max_combination_length - self._action_history = [] - - def loops_detected(self, action_key): - """Add action key to the history and detect loops. - - :param action_key: - :return: - """ - # """Added action key to the history and detect for loops""" - loops_detected = False - self._action_history.append(action_key) - for combination_length in range(1, self._max_combination_length + 1): - if self._is_combination_compatible(combination_length): - if self._detect_loops_for_combination_length(combination_length): - loops_detected = True - break - return loops_detected - - def _is_combination_compatible(self, combination_length): - """Check if combinations may exist. - - :param combination_length: - :return: - """ - if len(self._action_history) / combination_length >= self._max_action_loops: - is_compatible = True - else: - is_compatible = False - return is_compatible - - def _detect_loops_for_combination_length(self, combination_length): - """Detect loops for combination length. - - :param combination_length: - :return: - """ - reversed_history = self._action_history[::-1] - combinations = [ - reversed_history[x : x + combination_length] - for x in range(0, len(reversed_history), combination_length) - ][: self._max_action_loops] - is_loops_exist = True - for x, y in [combinations[x : x + 2] for x in range(0, len(combinations) - 1)]: - if x != y: - is_loops_exist = False - break - return is_loops_exist From e23bb20840ceed70bc1125d16a051620f83b4cc0 Mon Sep 17 00:00:00 2001 From: anthony Date: Wed, 26 Jun 2019 23:23:41 +0300 Subject: [PATCH 44/59] replace Action map OrderedDict with ActionMap class --- .../cli/command_template/command_template.py | 31 ++++++---- .../command_template_executor.py | 60 +++++++------------ cloudshell/cli/service/cli_service_impl.py | 5 +- cloudshell/cli/service/command_mode.py | 7 ++- cloudshell/cli/session/expect_session.py | 7 +-- cloudshell/cli/session/telnet_session.py | 25 ++++---- 6 files changed, 57 insertions(+), 78 deletions(-) diff --git a/cloudshell/cli/command_template/command_template.py b/cloudshell/cli/command_template/command_template.py index 6900b0a..5034eba 100644 --- a/cloudshell/cli/command_template/command_template.py +++ b/cloudshell/cli/command_template/command_template.py @@ -1,19 +1,21 @@ +from collections import OrderedDict import re from collections import OrderedDict +from cloudshell.cli.service.action_map import ActionMap + class CommandTemplate: def __init__(self, command, action_map=None, error_map=None): """Command Template. :type command: str - :type action_map: dict - :param error_map: expected error map with subclass of CommandExecutionException - or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] # noqa: E501 + :param cloudshell.cli.service.action_map.ActionMap action_map: + :param error_map: expected error map with subclass of CommandExecutionException or str + :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] """ self._command = command - self._action_map = action_map or OrderedDict() + self._action_map = action_map or ActionMap() self._error_map = error_map or OrderedDict() @property @@ -34,14 +36,17 @@ def error_map(self): # ToDo: Needs to be reviewed def get_command(self, **kwargs): - action_map = OrderedDict(kwargs.get("action_map", None) or OrderedDict()) - action_map.update(self._action_map) - error_map = OrderedDict(self._error_map) - error_map.update(kwargs.get("error_map", None) or OrderedDict()) + # todo: verify action map creation + action_map = kwargs.get('action_map') or ActionMap() + action_map.extend(self.action_map) + + error_map = kwargs.get("error_map") or OrderedDict() + error_map.update(self.error_map) + return { - "command": self.prepare_command(**kwargs), - "action_map": action_map, - "error_map": error_map, + 'command': self.prepare_command(**kwargs), + 'action_map': action_map, + 'error_map': error_map } def prepare_command(self, **kwargs): @@ -52,7 +57,7 @@ def prepare_command(self, **kwargs): cmd = re.sub(r"\[[^[]*?{{{key}}}.*?\]".format(key=key), r"", cmd) if not cmd: - raise Exception(self.__class__.__name__, "Unable to prepare command") + raise Exception(self.__class__.__name__, 'Unable to prepare command') cmd = re.sub(r"\s+", " ", cmd).strip(" \t\n\r") result = re.sub(r"\[|\]", "", cmd).format(**kwargs) diff --git a/cloudshell/cli/command_template/command_template_executor.py b/cloudshell/cli/command_template/command_template_executor.py index 049129a..16bddc7 100644 --- a/cloudshell/cli/command_template/command_template_executor.py +++ b/cloudshell/cli/command_template/command_template_executor.py @@ -1,8 +1,11 @@ from collections import OrderedDict +from cloudshell.cli.service.action_map import ActionMap + class CommandTemplateExecutor(object): """Execute command template using cli service.""" + """Execute command template using cli service""" def __init__( self, @@ -14,57 +17,34 @@ def __init__( ): """Initialize Command template executor. - :param cli_service: - :type cli_service: CliService - :param command_template: - :type command_template: cloudshell.cli.command_template.command_template.CommandTemplate # noqa: E501 - :param error_map: expected error map with subclass of CommandExecutionException - or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] # noqa: E501 + :param cloudshell.cli.service.cli_service.CliService cli_service: + :param cloudshell.cli.command_template.command_template.CommandTemplate command_template: + :param cloudshell.cli.service.action_map.ActionMap action_map: + :param error_map: expected error map with subclass of CommandExecutionException or str + :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :return: """ + self._cli_service = cli_service self._command_template = command_template - self._action_map = action_map or OrderedDict() - self._error_map = error_map or OrderedDict() - self._optional_kwargs = optional_kwargs - @property - def action_map(self): - """Return updated action.""" - action_map = self._action_map.copy() - action_map.update(self._command_template.action_map) - return action_map + self._action_map = action_map or ActionMap() + self._action_map.extend(command_template.action_map) - @property - def error_map(self): - error_map = self._error_map.copy() - error_map.update(self._command_template.error_map) - return error_map + self._error_map = error_map or OrderedDict() + self._error_map.update(command_template.error_map) - @property - def optional_kwargs(self): - return self._optional_kwargs + self._optional_kwargs = optional_kwargs def execute_command(self, **command_kwargs): """Execute command. - :param command_kwargs: + :param dict command_kwargs: :return: Command output :rtype: str """ command = self._command_template.prepare_command(**command_kwargs) - return self._cli_service.send_command( - command, - action_map=self.action_map, - error_map=self.error_map, - **self.optional_kwargs - ) - - def update_action_map(self, action_map): - self._action_map.update(action_map) - - def update_error_map(self, error_map): - self._error_map.update(error_map) - - def update_optional_kwargs(self, **optional_kwargs): - self.optional_kwargs.update(optional_kwargs) + return self._cli_service.send_command(command, + action_map=self._action_map, + error_map=self._error_map, + **self._optional_kwargs) diff --git a/cloudshell/cli/service/cli_service_impl.py b/cloudshell/cli/service/cli_service_impl.py index 806d11f..606b002 100644 --- a/cloudshell/cli/service/cli_service_impl.py +++ b/cloudshell/cli/service/cli_service_impl.py @@ -117,9 +117,8 @@ def send_command( :param command: :param expected_string: :param action_map: - :param error_map: expected error map with subclass of CommandExecutionException - or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] # noqa: E501 + :param error_map: expected error map with subclass of CommandExecutionException or str + :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] :param logger: :param remove_prompt: :param args: diff --git a/cloudshell/cli/service/command_mode.py b/cloudshell/cli/service/command_mode.py index 5ad4ee1..b1f53b2 100755 --- a/cloudshell/cli/service/command_mode.py +++ b/cloudshell/cli/service/command_mode.py @@ -1,5 +1,6 @@ import re +from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.service.cli_exception import CliException from cloudshell.cli.service.node import Node @@ -35,7 +36,7 @@ def __init__( :param exit_command: Command used to exit from this mode :type exit_command: str :param enter_actions: Actions which needs to be done when entering this mode - :param enter_action_map: Enter expected actions + :param cloudshell.cli.service.action_map.ActionMap enter_action_map: Enter expected actions :type enter_action_map: dict :param enter_error_map: expected error map with subclass of CommandExecutionException or str @@ -53,9 +54,9 @@ def __init__( if not enter_error_map: enter_error_map = {} if not exit_action_map: - exit_action_map = {} + exit_action_map = ActionMap() if not enter_action_map: - enter_action_map = {} + enter_action_map = ActionMap() super(CommandMode, self).__init__() self._prompt = prompt diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 1e50723..81b7fac 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -212,14 +212,13 @@ def hardware_expect( :param command: command to send :param expected_string: expected string :param logger: logger - :param action_map: dict with {re_str: action} to trigger some action on received string + :param action_map: ActionMap :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, CommandExecutionException|str] :param timeout: session timeout :param retries: maximal retries count - :param remove_command_from_output: In some switches the output string includes - the command which was called. The flag used to verify whether the the - command string removed from the output string. + :param remove_command_from_output: In some switches the output string includes the command which was called. + The flag used to verify whether the the command string removed from the output string. :return: :rtype: str """ diff --git a/cloudshell/cli/session/telnet_session.py b/cloudshell/cli/session/telnet_session.py index b5bd68a..8617598 100644 --- a/cloudshell/cli/session/telnet_session.py +++ b/cloudshell/cli/session/telnet_session.py @@ -1,7 +1,8 @@ import socket import telnetlib -from collections import OrderedDict +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.session.connection_params import ConnectionParams from cloudshell.cli.session.expect_session import ExpectSession from cloudshell.cli.session.session_exceptions import ( @@ -58,20 +59,14 @@ def __del__(self): self.disconnect() def _connect_actions(self, prompt, logger): - action_map = OrderedDict() - action_map[ - "[Ll]ogin:|[Uu]ser:|[Uu]sername:" - ] = lambda session, logger: session.send_line(session.username, logger) - action_map["[Pp]assword:"] = lambda session, logger: session.send_line( - session.password, logger - ) - self.hardware_expect( - None, - expected_string=prompt, - timeout=self._timeout, - logger=logger, - action_map=action_map, - ) + action_map = ActionMap(actions=[Action(pattern="[Ll]ogin:|[Uu]ser:|[Uu]sername:", + callback=lambda session, logger: + session.send_line(session.username, logger)), + Action(pattern="[Pp]assword:", + callback=lambda session, logger: + session.send_line(session.password, logger))]) + + self.hardware_expect(None, expected_string=prompt, timeout=self._timeout, logger=logger, action_map=action_map) self._on_session_start(logger) def _initialize_session(self, prompt, logger): From 89c403a719f7a873043fb10bc2b72659068da9ac Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 12:45:40 +0300 Subject: [PATCH 45/59] fix error map initialization --- cloudshell/cli/session/expect_session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 81b7fac..5842bf8 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,6 +1,7 @@ import re import time from abc import ABCMeta, abstractmethod +from collections import OrderedDict from cloudshell.cli.service.action_map import ActionMap from cloudshell.cli.service.action_map import ActionLoopDetector @@ -226,7 +227,7 @@ def hardware_expect( action_map = ActionMap() if not error_map: - error_map = ActionMap() + error_map = OrderedDict() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout From 025c3eb2627f7161dcfc823005096a7519a1340d Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 12:58:42 +0300 Subject: [PATCH 46/59] replace "super(Class, self)" calls with "super()" call --- cloudshell/cli/service/action_map.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 09c644d..2f39624 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -30,9 +30,7 @@ def __repr__(self): :rtype: str """ - return "{} pattern: {}, execute once: {}".format(super(Action, self).__repr__(), - self.pattern, - self.execute_once) + return "{} pattern: {}, execute once: {}".format(super().__repr__(), self.pattern, self.execute_once) def match(self, output): """ @@ -137,9 +135,7 @@ def __repr__(self): :rtype: str """ - return "{} matched patterns: {}, actions: {}".format(super(ActionMap, self).__repr__(), - self.matched_patterns, - self.actions) + return "{} matched patterns: {}, actions: {}".format(super().__repr__(), self.matched_patterns, self.actions) class ActionLoopDetector(object): From ce10499e766268d51c5b288aa25679243ef66188 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 13:23:11 +0300 Subject: [PATCH 47/59] use f-Strings instead of str.format --- cloudshell/cli/service/action_map.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 2f39624..fd98a92 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -30,7 +30,7 @@ def __repr__(self): :rtype: str """ - return "{} pattern: {}, execute once: {}".format(super().__repr__(), self.pattern, self.execute_once) + return f"{super().__repr__()} pattern: {self.pattern}, execute once: {self.execute_once}" def match(self, output): """ @@ -104,13 +104,13 @@ def __call__(self, session, logger, output, check_action_loop_detector, action_l """ for action in self.active_actions: if action.match(output): - logger.debug("Matched Action with pattern: {}".format(action.pattern)) + logger.debug(f"Matched Action with pattern: {action.pattern}") if check_action_loop_detector: - logger.debug("Checking loops fro Action with pattern : {}".format(action.pattern)) + logger.debug(f"Checking loops for Action with pattern : {action.pattern}") if action_loop_detector.loops_detected(action.pattern): - logger.error("Loops detected for action patter: {}".format(action.pattern)) + logger.error(f"Loops detected for action patter: {action.pattern}") raise SessionLoopDetectorException("Expected actions loops detected") action(session, logger) @@ -128,14 +128,14 @@ def __add__(self, other): if isinstance(other, type(self)): return ActionMap(actions=self.actions + other.actions) - raise TypeError("unsupported operand type(s) for +: '{}' and '{}'".format(type(self), type(other))) + raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") def __repr__(self): """ :rtype: str """ - return "{} matched patterns: {}, actions: {}".format(super().__repr__(), self.matched_patterns, self.actions) + return f"{super().__repr__()} matched patterns: {self.matched_patterns}, actions: {self.actions}" class ActionLoopDetector(object): From 75b31e13b2b37f3dedde340c71bf3dfd306764ea Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 16:39:41 +0300 Subject: [PATCH 48/59] add unit tests for the "action_map" module --- tests/cli/service/__init__.py | 0 tests/cli/service/test_action_map.py | 132 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/cli/service/__init__.py create mode 100644 tests/cli/service/test_action_map.py diff --git a/tests/cli/service/__init__.py b/tests/cli/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/service/test_action_map.py b/tests/cli/service/test_action_map.py new file mode 100644 index 0000000..ad72e56 --- /dev/null +++ b/tests/cli/service/test_action_map.py @@ -0,0 +1,132 @@ +import unittest +from unittest import mock + +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap + + +class TestAction(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.callback = mock.MagicMock() + self.action = Action(pattern="test pattern", callback=self.callback) + + def test_call(self): + """Check that method will call callback function with session and logger as attributes""" + # act + self.action(session=self.session, logger=self.logger) + # verify + self.callback.assert_called_once_with(self.session, self.logger) + + @mock.patch("cloudshell.cli.service.action_map.re") + def test_match_return_true(self, re): + """Check that method will return True if output matches pattern""" + re.search.return_value = True + output = "test output" + # act + result = self.action.match(output) + # verify + self.assertTrue(result) + + @mock.patch("cloudshell.cli.service.action_map.re") + def test_match_return_false(self, re): + """Check that method will return False if output doesn't match pattern""" + re.search.return_value = False + output = "test output" + # act + result = self.action.match(output) + # verify + self.assertFalse(result) + + +class TestActionMap(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.callback = mock.MagicMock() + self.actions = [Action(pattern="[Pp]attern 1", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback, execute_once=True), + Action(pattern="[Pp]attern 3", callback=self.callback, execute_once=True)] + + self.action_map = ActionMap(actions=self.actions) + + def test_actions(self): + """Check that method will return all actions""" + action1, action2, action3 = self.actions + self.action_map.matched_patterns = {action1.pattern, action2.pattern} + # verify + self.assertEqual(self.action_map.actions, [action1, action2, action3]) + + def test_active_actions(self): + """Check that method will return only active actions""" + action1, action2, action3 = self.actions + self.action_map.matched_patterns = {action1.pattern, action2.pattern} + # verify + self.assertEqual(self.action_map.active_actions, [action1, action3]) + + def test_extend(self): + """Check that extend will add new actions and will not override existing one""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + self.action_map.extend(action_map=default_action_map) + + # verify + self.assertEqual(self.action_map.matched_patterns, {default_action1.pattern, + default_action2.pattern, + action3.pattern}) + + self.assertEqual(self.action_map.actions, [action1, action2, action3, default_action1]) + + def test_extend_with_override_true(self): + """Check that extend will add new actions and will not override existing one""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + self.action_map.extend(action_map=default_action_map, override=True) + + # verify + self.assertEqual(self.action_map.matched_patterns, {default_action1.pattern, + default_action2.pattern, + action3.pattern}) + + self.assertEqual(self.action_map.actions, [action1, default_action2, action3, default_action1]) + + def test_add(self): + """Check that __add__ method will create new ActionMap""" + default_actions = [Action(pattern="[Pp]attern 4", callback=self.callback), + Action(pattern="[Pp]attern 2", callback=self.callback)] + + default_action_map = ActionMap(actions=default_actions) + + default_action1, default_action2 = default_actions + action1, action2, action3 = self.actions + + default_action_map.matched_patterns = {default_action1.pattern, default_action2.pattern} + self.action_map.matched_patterns = {action3.pattern} + + # act + result = self.action_map + default_action_map + + # verify + self.assertEqual(result.matched_patterns, set()) + self.assertEqual(result.actions, [action1, action2, action3, default_action1]) From 5337d03cf7ee27368e63f1d17ae60ecdcb5df3c0 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 2 Jul 2019 16:59:04 +0300 Subject: [PATCH 49/59] update ActionMap.__add__ method logic --- cloudshell/cli/service/action_map.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index fd98a92..8391447 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -126,7 +126,10 @@ def __add__(self, other): :rtype: ActionMap """ if isinstance(other, type(self)): - return ActionMap(actions=self.actions + other.actions) + actions = self.actions + [action for action in other.actions if action.pattern not in + [action.pattern for action in self.actions]] + + return ActionMap(actions=actions) raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") From 452c7d6569b1c2bedf861c09531f81cc8a887beb Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 15:50:58 +0300 Subject: [PATCH 50/59] add "Error" and "ErrorMap" classes --- .../cli/command_template/command_template.py | 16 ++- .../command_template_executor.py | 10 +- cloudshell/cli/service/cli_service_impl.py | 2 +- cloudshell/cli/service/command_mode.py | 27 ++-- cloudshell/cli/service/error_map.py | 118 ++++++++++++++++++ cloudshell/cli/session/expect_session.py | 4 +- 6 files changed, 141 insertions(+), 36 deletions(-) create mode 100644 cloudshell/cli/service/error_map.py diff --git a/cloudshell/cli/command_template/command_template.py b/cloudshell/cli/command_template/command_template.py index 5034eba..76dbdac 100644 --- a/cloudshell/cli/command_template/command_template.py +++ b/cloudshell/cli/command_template/command_template.py @@ -1,8 +1,7 @@ -from collections import OrderedDict import re -from collections import OrderedDict from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap class CommandTemplate: @@ -11,18 +10,17 @@ def __init__(self, command, action_map=None, error_map=None): :type command: str :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: """ self._command = command self._action_map = action_map or ActionMap() - self._error_map = error_map or OrderedDict() + self._error_map = error_map or ErrorMap() @property def action_map(self): """Property for action map. - :rtype: OrderedDict() + :rtype: cloudshell.cli.service.action_map.ActionMap """ return self._action_map @@ -30,7 +28,7 @@ def action_map(self): def error_map(self): """Property for error map. - :rtype: OrderedDict + :rtype: cloudshell.cli.service.error_map.ErrorMap """ return self._error_map @@ -40,8 +38,8 @@ def get_command(self, **kwargs): action_map = kwargs.get('action_map') or ActionMap() action_map.extend(self.action_map) - error_map = kwargs.get("error_map") or OrderedDict() - error_map.update(self.error_map) + error_map = kwargs.get("error_map") or ErrorMap() + error_map.extend(self.error_map) return { 'command': self.prepare_command(**kwargs), diff --git a/cloudshell/cli/command_template/command_template_executor.py b/cloudshell/cli/command_template/command_template_executor.py index 16bddc7..e4d5b20 100644 --- a/cloudshell/cli/command_template/command_template_executor.py +++ b/cloudshell/cli/command_template/command_template_executor.py @@ -1,6 +1,5 @@ -from collections import OrderedDict - from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap class CommandTemplateExecutor(object): @@ -20,8 +19,7 @@ def __init__( :param cloudshell.cli.service.cli_service.CliService cli_service: :param cloudshell.cli.command_template.command_template.CommandTemplate command_template: :param cloudshell.cli.service.action_map.ActionMap action_map: - :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] + :param cloudshell.cli.service.error_map.ErrorMap error_map: :return: """ @@ -31,8 +29,8 @@ def __init__( self._action_map = action_map or ActionMap() self._action_map.extend(command_template.action_map) - self._error_map = error_map or OrderedDict() - self._error_map.update(command_template.error_map) + self._error_map = error_map or ErrorMap() + self._error_map.extend(command_template.error_map) self._optional_kwargs = optional_kwargs diff --git a/cloudshell/cli/service/cli_service_impl.py b/cloudshell/cli/service/cli_service_impl.py index 606b002..3f4af0e 100644 --- a/cloudshell/cli/service/cli_service_impl.py +++ b/cloudshell/cli/service/cli_service_impl.py @@ -116,7 +116,7 @@ def send_command( :param command: :param expected_string: - :param action_map: + :param cloudshell.cli.service.action_map.ActionMap action_map: :param error_map: expected error map with subclass of CommandExecutionException or str :type error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] :param logger: diff --git a/cloudshell/cli/service/command_mode.py b/cloudshell/cli/service/command_mode.py index b1f53b2..7d9e6e8 100755 --- a/cloudshell/cli/service/command_mode.py +++ b/cloudshell/cli/service/command_mode.py @@ -1,6 +1,7 @@ import re from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.service.cli_exception import CliException from cloudshell.cli.service.node import Node @@ -29,30 +30,20 @@ def __init__( ): """Initialize Command Mode. - :param prompt: Prompt of this mode - :type prompt: str - :param enter_command: Command used to enter this mode - :type enter_command: str - :param exit_command: Command used to exit from this mode - :type exit_command: str + :param str prompt: Prompt of this mode + :param str enter_command: Command used to enter this mode + :param str exit_command: Command used to exit from this mode :param enter_actions: Actions which needs to be done when entering this mode :param cloudshell.cli.service.action_map.ActionMap enter_action_map: Enter expected actions - :type enter_action_map: dict - :param enter_error_map: expected error map with subclass of - CommandExecutionException or str - :type enter_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] # noqa: E501 - :param exit_action_map: - :type exit_action_map: dict - :param exit_error_map: expected error map with subclass of - CommandExecutionException or str - :type exit_error_map: dict[str, cloudshell.cli.session.session_exceptions.CommandExecutionException|str] # noqa: E501 - :param + :param cloudshell.cli.service.error_map.ErrorMap enter_error_map: + :param cloudshell.cli.service.action_map.ActionMap exit_action_map: + :param cloudshell.cli.service.error_map.ErrorMap exit_error_map: :param parent_mode: Connect parent mode """ if not exit_error_map: - exit_error_map = {} + exit_error_map = ErrorMap() if not enter_error_map: - enter_error_map = {} + enter_error_map = ErrorMap() if not exit_action_map: exit_action_map = ActionMap() if not enter_action_map: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py new file mode 100644 index 0000000..2b6b4e4 --- /dev/null +++ b/cloudshell/cli/service/error_map.py @@ -0,0 +1,118 @@ +from collections import OrderedDict +import re + +from cloudshell.cli.session.session_exceptions import CommandExecutionException + + +class Error: + def __init__(self, pattern, error): + """ + + :param str pattern: + :param str error: + """ + self.pattern = pattern + self.error = error + + def __call__(self): + """ + + :raises: CommandExecutionException + """ + if isinstance(self.error, CommandExecutionException): + raise self.error + + raise CommandExecutionException(f"Session returned '{self.error}'") + + def __repr__(self): + """ + + :rtype: str + """ + return f"{super().__repr__()} pattern: {self.pattern}, error: {self.error}" + + def match(self, output): + """ + + :param str output: + :rtype: bool + """ + return bool(re.search(self.pattern, output, re.DOTALL)) + + +class ErrorMap: + def __init__(self, errors=None): + """ + + :param list[Error] errors: + """ + if errors is None: + errors = [] + + self._errors_dict = OrderedDict([(error.pattern, error) for error in errors]) + + @property + def errors(self): + """ + + :rtype: list[Error] + """ + return list(self._errors_dict.values()) + + def add(self, error): + """ + + :param Error error: + :return: + """ + self._errors_dict[error.pattern] = error + + def extend(self, error_map, override=False): + """ + + :param ErrorMap error_map: + :param bool override: + :return: + """ + for error in error_map.errors: + if not override and error.pattern in self._errors_dict: + continue + self.add(error) + + def __call__(self, output, logger): + """ + + :param str output: + :param logging.Logger logger: + :rtype: bool + """ + + for error in self.errors: + if error.match(output): + logger.debug(f"Matched Error with pattern: {error.pattern}") + + if isinstance(error, CommandExecutionException): + raise error + else: + raise CommandExecutionException('Session returned \'{}\''.format(error)) + + def __add__(self, other): + """ + + :param other: + :rtype: ActionMap + """ + if isinstance(other, type(self)): + errors = self.errors + [error for error in other.errors if error.pattern not in + [error.pattern for error in self.errors]] + + return ErrorMap(errors=errors) + + raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") + + def __repr__(self): + """ + + :rtype: str + """ + return f"{super().__repr__()} errors: {self.errors}" diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 5842bf8..f4b4da8 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -1,9 +1,9 @@ import re import time from abc import ABCMeta, abstractmethod -from collections import OrderedDict from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.service.action_map import ActionLoopDetector from cloudshell.cli.session.helper.normalize_buffer import normalize_buffer from cloudshell.cli.session.session import Session @@ -227,7 +227,7 @@ def hardware_expect( action_map = ActionMap() if not error_map: - error_map = OrderedDict() + error_map = ErrorMap() retries = retries or self._max_loop_retries empty_loop_timeout = empty_loop_timeout or self._empty_loop_timeout From 5b109350e646d8630ad24d59016effd3c62ce89c Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 15:52:53 +0300 Subject: [PATCH 51/59] small fixes for the action_map module --- cloudshell/cli/service/action_map.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 8391447..f210f40 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -4,7 +4,7 @@ from cloudshell.cli.session.session_exceptions import SessionLoopDetectorException -class Action(object): +class Action: def __init__(self, pattern, callback, execute_once=False): """ @@ -41,7 +41,7 @@ def match(self, output): return bool(re.search(self.pattern, output, re.DOTALL)) -class ActionMap(object): +class ActionMap: def __init__(self, actions=None): """ @@ -59,7 +59,7 @@ def actions(self): :rtype: list[Action] """ - return [action for action in self._actions_dict.values()] + return list(self._actions_dict.values()) @property def active_actions(self): @@ -141,7 +141,7 @@ def __repr__(self): return f"{super().__repr__()} matched patterns: {self.matched_patterns}, actions: {self.actions}" -class ActionLoopDetector(object): +class ActionLoopDetector: """Help to detect loops for action combinations""" def __init__(self, max_loops, max_combination_length): From 29b98ba919720def305ef68505432fef1680e16f Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 16:19:51 +0300 Subject: [PATCH 52/59] update ErrorMap usage in the "hardware_expect" method --- cloudshell/cli/session/expect_session.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index f4b4da8..509c758 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -302,16 +302,7 @@ def hardware_expect( result_output = "".join(output_list) - for error_pattern, error in error_map.items(): - result_match = re.search(error_pattern, result_output, re.DOTALL) - - if result_match: - if isinstance(error, CommandExecutionException): - raise error - else: - raise CommandExecutionException( - "Session returned '{}'".format(error) - ) + error_map(output=result_output, logger=logger) # Read buffer to the end. Useful when expected_string isn't last in buffer result_output += self._clear_buffer(self._clear_buffer_timeout, logger) From fff5750946ccf77f9507ec9a57172e2f5160ad51 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 16:42:41 +0300 Subject: [PATCH 53/59] fix unittest and bugs for the "error_map" module --- cloudshell/cli/service/error_map.py | 8 ++------ tests/cli/session/test_expect_session.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 2b6b4e4..16e2fe3 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -9,7 +9,7 @@ def __init__(self, pattern, error): """ :param str pattern: - :param str error: + :param str|CommandExecutionException error: """ self.pattern = pattern self.error = error @@ -90,11 +90,7 @@ def __call__(self, output, logger): for error in self.errors: if error.match(output): logger.debug(f"Matched Error with pattern: {error.pattern}") - - if isinstance(error, CommandExecutionException): - raise error - else: - raise CommandExecutionException('Session returned \'{}\''.format(error)) + error() def __add__(self, other): """ diff --git a/tests/cli/session/test_expect_session.py b/tests/cli/session/test_expect_session.py index 7a110a6..5ae75ba 100644 --- a/tests/cli/session/test_expect_session.py +++ b/tests/cli/session/test_expect_session.py @@ -1,6 +1,9 @@ -from collections import OrderedDict from unittest import TestCase +from cloudshell.cli.service.action_map import Action +from cloudshell.cli.service.action_map import ActionMap +from cloudshell.cli.service.error_map import Error +from cloudshell.cli.service.error_map import ErrorMap from cloudshell.cli.session.expect_session import ActionLoopDetector, ExpectSession from cloudshell.cli.session.session_exceptions import ( CommandExecutionException, @@ -275,10 +278,8 @@ def test_hardware_expect_action_map_call( receive_all.side_effect = side_effect normalize_buffer.side_effect = side_effect test_func = Mock() - action_map = OrderedDict({fake_out: test_func}) - self._instance.hardware_expect( - command, expected_string, self._logger, action_map=action_map - ) + action_map = ActionMap(actions=[Action(pattern=fake_out, callback=test_func)]) + self._instance.hardware_expect(command, expected_string, self._logger, action_map=action_map) test_func.assert_called_once_with(self._instance, self._logger) @patch("cloudshell.cli.session.expect_session.ExpectSession.send_line") @@ -294,7 +295,7 @@ def test_hardware_expect_error_map_call( expected_string = "test_string" receive_all.return_value = expected_string normalize_buffer.return_value = expected_string - error_map = OrderedDict({expected_string: "test_error"}) + error_map = ErrorMap(errors=[Error(pattern=expected_string, error='test_error')]) exception = CommandExecutionException with self.assertRaises(exception): self._instance.hardware_expect( @@ -317,7 +318,8 @@ class TestException(CommandExecutionException): expected_string = "test_string" receive_all.return_value = expected_string normalize_buffer.return_value = expected_string - error_map = OrderedDict({expected_string: TestException("test_error")}) + error_map = ErrorMap(errors=[Error(pattern=expected_string, error=TestException('test_error'))]) + with self.assertRaises(TestException): self._instance.hardware_expect( command, expected_string, self._logger, error_map=error_map From 283021963ed31c40cc872e51626dba9608a6b909 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 17:17:37 +0300 Subject: [PATCH 54/59] add unit tests for the "error_map" module --- tests/cli/service/test_error_map.py | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/cli/service/test_error_map.py diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py new file mode 100644 index 0000000..aa9b89e --- /dev/null +++ b/tests/cli/service/test_error_map.py @@ -0,0 +1,87 @@ +import unittest +from unittest import mock + +from cloudshell.cli.service.error_map import Error +from cloudshell.cli.service.error_map import ErrorMap +from cloudshell.cli.session.session_exceptions import CommandExecutionException + + +class TestError(unittest.TestCase): + def setUp(self): + self.session = mock.MagicMock() + self.logger = mock.MagicMock() + self.error_msg = "error message" + self.error = Error(pattern="test pattern", error=self.error_msg) + + def test_call(self): + """Check that method will raise CommandExecutionException""" + with self.assertRaisesRegex(CommandExecutionException, self.error_msg): + self.error() + + @mock.patch("cloudshell.cli.service.error_map.re") + def test_match_return_true(self, re): + """Check that method will return True if output matches pattern""" + re.search.return_value = True + output = "test output" + # act + result = self.error.match(output) + # verify + self.assertTrue(result) + + @mock.patch("cloudshell.cli.service.error_map.re") + def test_match_return_false(self, re): + """Check that method will return False if output doesn't match pattern""" + re.search.return_value = False + output = "test output" + # act + result = self.error.match(output) + # verify + self.assertFalse(result) + + +class TestErrorMap(unittest.TestCase): + def setUp(self): + self.logger = mock.MagicMock() + self.errors = [Error(pattern="[Pp]attern 1", error="error 1"), + Error(pattern="[Pp]attern 2", error="error 2")] + + self.error_map = ErrorMap(errors=self.errors) + + def test_errors(self): + """Check that method will return errors""" + # verify + self.assertEqual(self.error_map.errors, self.errors) + + def test_extend(self): + """Check that extend will add new errors and will not override existing one""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + self.error_map.extend(error_map=default_error_map) + # verify + self.assertEqual(self.error_map.errors, [error1, error2, default_error1]) + + def test_extend_with_override_true(self): + """Check that extend will add new errors and will not override existing one""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + self.error_map.extend(error_map=default_error_map, override=True) + # verify + self.assertEqual(self.error_map.errors, [error1, default_error2, default_error1]) + + def test_add(self): + """Check that __add__ method will create new ErrorMap""" + default_error1 = Error(pattern="[Pp]attern 4", error="error 4") + default_error2 = Error(pattern="[Pp]attern 2", error="error 2") + default_error_map = ErrorMap(errors=[default_error1, default_error2]) + error1, error2 = self.errors + # act + result = self.error_map + default_error_map + # verify + self.assertEqual(result.errors, [error1, error2, default_error1]) + From e2bb755244d401d2dd965148ef39bd86a55a587f Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 8 Jul 2019 17:52:51 +0300 Subject: [PATCH 55/59] use compiled re pattern in the "error_map" and "action_map" modules --- cloudshell/cli/service/action_map.py | 3 ++- cloudshell/cli/service/error_map.py | 3 ++- tests/cli/service/test_action_map.py | 12 ++++-------- tests/cli/service/test_error_map.py | 12 ++++-------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index f210f40..2eb207e 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -13,6 +13,7 @@ def __init__(self, pattern, callback, execute_once=False): :param bool execute_once: """ self.pattern = pattern + self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.callback = callback self.execute_once = execute_once @@ -38,7 +39,7 @@ def match(self, output): :param str output: :rtype: bool """ - return bool(re.search(self.pattern, output, re.DOTALL)) + return bool(self.compiled_pattern.search(output)) class ActionMap: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 16e2fe3..4b55e08 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -12,6 +12,7 @@ def __init__(self, pattern, error): :param str|CommandExecutionException error: """ self.pattern = pattern + self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.error = error def __call__(self): @@ -37,7 +38,7 @@ def match(self, output): :param str output: :rtype: bool """ - return bool(re.search(self.pattern, output, re.DOTALL)) + return bool(self.compiled_pattern.search(output)) class ErrorMap: diff --git a/tests/cli/service/test_action_map.py b/tests/cli/service/test_action_map.py index ad72e56..8f849b0 100644 --- a/tests/cli/service/test_action_map.py +++ b/tests/cli/service/test_action_map.py @@ -19,21 +19,17 @@ def test_call(self): # verify self.callback.assert_called_once_with(self.session, self.logger) - @mock.patch("cloudshell.cli.service.action_map.re") - def test_match_return_true(self, re): + def test_match_return_true(self): """Check that method will return True if output matches pattern""" - re.search.return_value = True - output = "test output" + output = "test pattern" # act result = self.action.match(output) # verify self.assertTrue(result) - @mock.patch("cloudshell.cli.service.action_map.re") - def test_match_return_false(self, re): + def test_match_return_false(self): """Check that method will return False if output doesn't match pattern""" - re.search.return_value = False - output = "test output" + output = "missed pattern" # act result = self.action.match(output) # verify diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py index aa9b89e..2f2f425 100644 --- a/tests/cli/service/test_error_map.py +++ b/tests/cli/service/test_error_map.py @@ -18,21 +18,17 @@ def test_call(self): with self.assertRaisesRegex(CommandExecutionException, self.error_msg): self.error() - @mock.patch("cloudshell.cli.service.error_map.re") - def test_match_return_true(self, re): + def test_match_return_true(self): """Check that method will return True if output matches pattern""" - re.search.return_value = True - output = "test output" + output = "test pattern" # act result = self.error.match(output) # verify self.assertTrue(result) - @mock.patch("cloudshell.cli.service.error_map.re") - def test_match_return_false(self, re): + def test_match_return_false(self): """Check that method will return False if output doesn't match pattern""" - re.search.return_value = False - output = "test output" + output = "missed pattern" # act result = self.error.match(output) # verify From 20704630c58d139ceccdf4ec6ff4802f648c6c65 Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 17:57:35 +0300 Subject: [PATCH 56/59] move "__call__" function to "process" in ActionMap and ErrorMap classes --- cloudshell/cli/service/action_map.py | 2 +- cloudshell/cli/service/error_map.py | 2 +- cloudshell/cli/session/expect_session.py | 17 ++++++++--------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index 2eb207e..d5a4c23 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -93,7 +93,7 @@ def extend(self, action_map, override=False): self.matched_patterns |= action_map.matched_patterns - def __call__(self, session, logger, output, check_action_loop_detector, action_loop_detector): + def process(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ :param cloudshell.cli.session.expect_session.ExpectSession session: diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 4b55e08..6bf1bb1 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -80,7 +80,7 @@ def extend(self, error_map, override=False): continue self.add(error) - def __call__(self, output, logger): + def process(self, output, logger): """ :param str output: diff --git a/cloudshell/cli/session/expect_session.py b/cloudshell/cli/session/expect_session.py index 509c758..e0c64b4 100644 --- a/cloudshell/cli/session/expect_session.py +++ b/cloudshell/cli/session/expect_session.py @@ -215,7 +215,7 @@ def hardware_expect( :param logger: logger :param action_map: ActionMap :param error_map: expected error map with subclass of CommandExecutionException or str - :type error_map: dict[str, CommandExecutionException|str] + :type error_map: ErrorMap :param timeout: session timeout :param retries: maximal retries count :param remove_command_from_output: In some switches the output string includes the command which was called. @@ -281,11 +281,11 @@ def hardware_expect( output_list.append(output_str) is_correct_exit = True - action_matched = action_map(session=self, - logger=logger, - output=output_str, - check_action_loop_detector=check_action_loop_detector, - action_loop_detector=action_loop_detector) + action_matched = action_map.process(session=self, + logger=logger, + output=output_str, + check_action_loop_detector=check_action_loop_detector, + action_loop_detector=action_loop_detector) if action_matched: output_list.append(output_str) @@ -300,9 +300,8 @@ def hardware_expect( "Session Loop limit exceeded, {} loops".format(retries_count), ) - result_output = "".join(output_list) - - error_map(output=result_output, logger=logger) + result_output = ''.join(output_list) + error_map.process(output=result_output, logger=logger) # Read buffer to the end. Useful when expected_string isn't last in buffer result_output += self._clear_buffer(self._clear_buffer_timeout, logger) From a6bc634b8f5c53c6e3c7ac5baa3aa621811b6af8 Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:19:11 +0300 Subject: [PATCH 57/59] simplify ActionMap and ErrorMap "__add__" methods --- cloudshell/cli/service/action_map.py | 16 +++++++++------- cloudshell/cli/service/error_map.py | 10 +++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cloudshell/cli/service/action_map.py b/cloudshell/cli/service/action_map.py index d5a4c23..114e3e4 100644 --- a/cloudshell/cli/service/action_map.py +++ b/cloudshell/cli/service/action_map.py @@ -79,11 +79,12 @@ def add(self, action): """ self._actions_dict[action.pattern] = action - def extend(self, action_map, override=False): + def extend(self, action_map, override=False, extend_matched_patterns=True): """ :param ActionMap action_map: :param bool override: + :param bool extend_matched_patterns: :return: """ for action in action_map.actions: @@ -91,7 +92,8 @@ def extend(self, action_map, override=False): continue self.add(action) - self.matched_patterns |= action_map.matched_patterns + if extend_matched_patterns: + self.matched_patterns |= action_map.matched_patterns def process(self, session, logger, output, check_action_loop_detector, action_loop_detector): """ @@ -126,11 +128,11 @@ def __add__(self, other): :param other: :rtype: ActionMap """ - if isinstance(other, type(self)): - actions = self.actions + [action for action in other.actions if action.pattern not in - [action.pattern for action in self.actions]] - - return ActionMap(actions=actions) + action_map_class = type(self) + if isinstance(other, action_map_class): + action_map = action_map_class(actions=self.actions) + action_map.extend(other, extend_matched_patterns=False) + return action_map raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index 6bf1bb1..ba6857d 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -99,11 +99,11 @@ def __add__(self, other): :param other: :rtype: ActionMap """ - if isinstance(other, type(self)): - errors = self.errors + [error for error in other.errors if error.pattern not in - [error.pattern for error in self.errors]] - - return ErrorMap(errors=errors) + error_map_class = type(self) + if isinstance(other, error_map_class): + error_map = error_map_class(errors=self.errors) + error_map.extend(other) + return error_map raise TypeError(f"unsupported operand type(s) for +: '{type(self)}' and '{type(other)}'") From 671dd4c0e60edbca7bdd40cc93ec7114f27f5273 Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:48:02 +0300 Subject: [PATCH 58/59] pass "output" argument to the ErrorMap "__call__" method --- cloudshell/cli/service/error_map.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudshell/cli/service/error_map.py b/cloudshell/cli/service/error_map.py index ba6857d..5dc82c0 100644 --- a/cloudshell/cli/service/error_map.py +++ b/cloudshell/cli/service/error_map.py @@ -15,9 +15,10 @@ def __init__(self, pattern, error): self.compiled_pattern = re.compile(pattern=pattern, flags=re.DOTALL) self.error = error - def __call__(self): + def __call__(self, output): """ + :param str output: :raises: CommandExecutionException """ if isinstance(self.error, CommandExecutionException): @@ -91,7 +92,7 @@ def process(self, output, logger): for error in self.errors: if error.match(output): logger.debug(f"Matched Error with pattern: {error.pattern}") - error() + error(output) def __add__(self, other): """ From 7fb1d8f39e6235da61f56a5237f8fdb59c389c9d Mon Sep 17 00:00:00 2001 From: anthony Date: Fri, 12 Jul 2019 18:52:02 +0300 Subject: [PATCH 59/59] fix unit test for the ErrorMap class --- tests/cli/service/test_error_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/service/test_error_map.py b/tests/cli/service/test_error_map.py index 2f2f425..abb9640 100644 --- a/tests/cli/service/test_error_map.py +++ b/tests/cli/service/test_error_map.py @@ -16,7 +16,7 @@ def setUp(self): def test_call(self): """Check that method will raise CommandExecutionException""" with self.assertRaisesRegex(CommandExecutionException, self.error_msg): - self.error() + self.error(output="test output") def test_match_return_true(self): """Check that method will return True if output matches pattern"""