From eafe1db03e85a9f3a9f2e51bfeb2b66899e39cd2 Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Tue, 14 Jul 2026 16:16:55 +0530 Subject: [PATCH 1/6] lf_interop_zoom.py: fail fast and log context when a resource lookup misses get_resource_data() silently skipped a user_resource that wasn't found in the /resource/all response, leaving device_names/ports_list/user_list shorter than self.real_sta_list. That mismatch surfaces much later as a pandas length-mismatch crash in generate_report(), far from the actual cause. Flatten the resources list into a dict once and look up matches directly instead of a nested scan, and on a miss, log the offending resource plus the full resources dict, then abort immediately so the test fails at the point of the real problem instead of downstream. Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index 7fe972920..40b1d6e14 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -845,27 +845,30 @@ def get_resource_data(self): # Step 2: Match user-specified resources with available resources sequentially if self.user_resources: resources = response.get("resources", []) - for user_resource in self.user_resources: - found = False - for element in resources: - if user_resource in element: - resource_values = element[user_resource] - eid = resource_values["eid"] - resource_ip = resource_values["ctrl-ip"] - hostname = resource_values["hostname"] - user = resource_values["user"] - - self.device_names.append(hostname) - self.ports_list.append({"eid": eid, "ctrl-ip": resource_ip}) - self.user_list.append(user) - - found = True - break + lf_resources_dict = {} + for element in resources: + lf_resources_dict.update(element) - if not found: - logger.warning( - f"Resource {user_resource} not found in LANforge response" + for user_resource in self.user_resources: + if user_resource in lf_resources_dict: + resource_values = lf_resources_dict[user_resource] + eid = resource_values["eid"] + resource_ip = resource_values["ctrl-ip"] + hostname = resource_values["hostname"] + user = resource_values["user"] + + self.device_names.append(hostname) + self.ports_list.append({"eid": eid, "ctrl-ip": resource_ip}) + self.user_list.append(user) + else: + logger.error( + f"Resource {user_resource} not found in LANforge response. Aborting test." + ) + logger.info( + "LANforge resources fetched from /resource/all:\n%s", + json.dumps(lf_resources_dict, indent=2, default=str), ) + exit(1) def get_ports_data(self): self.gen_ports_list = [] From 40c39cf7334feda8b746eb98671823d9d8b8feaa Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Tue, 14 Jul 2026 17:08:41 +0530 Subject: [PATCH 2/6] lf_interop_zoom.py: guard remaining LANforge/Zoom API calls against failure get_ports_data(), get_interop_data(), and select_real_devices() each used a self.json_get() response without checking for None first, crashing with an unhelpful KeyError/TypeError deep in dict access if LANforge was unreachable or returned no data. Add a None check with a clear log message and exit(1) at each call site instead. get_access_token() raised a bare Exception on both a failed HTTP request and a non-200 response, with no logging at either point. Wrap the request in try/except, log the account_id and failure reason in both cases, and return None instead of raising so callers can handle the failure explicitly rather than relying on an uncaught exception. get_live_data() and get_final_qos_data() now check for a None token before using it, logging and bailing out early instead of proceeding to call get_participants_qos() with an invalid Bearer token. Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index 40b1d6e14..eb3223002 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -879,6 +879,9 @@ def get_ports_data(self): # Step 3: Retrieve port information response_port = self.json_get("/port/all") + if response_port is None: + logger.error("GET /port/all returned no response from LANforge. Aborting test.") + exit(1) # Step 4: Match ports associated with retrieved resources in the order of ports_list for port_entry in self.ports_list: @@ -924,6 +927,9 @@ def get_ports_data(self): def get_interop_data(self): interop_data = self.json_get("/adb") + if interop_data is None: + logger.error("GET /adb returned no response from LANforge. Aborting test.") + exit(1) interop_mobile_data = interop_data.get("devices", {}) self.serial_list = [] self.lanforge_port_list = [] @@ -1246,6 +1252,9 @@ def select_real_devices(self, real_device_obj, real_sta_list=None): self.real_sta_list, _, _ = real_device_obj.query_user() else: interface_data = self.json_get("/port/all") + if interface_data is None: + logger.error("GET /port/all returned no response from LANforge. Aborting test.") + exit(1) interfaces = interface_data["interfaces"] final_device_list = [] # Initialize the list @@ -1396,16 +1405,27 @@ def get_signal_and_channel_data_dict(self): def get_access_token(self, account_id, client_id, client_secret): token_url = f"https://zoom.us/oauth/token?grant_type=account_credentials&account_id={account_id}" - response = requests.post( - token_url, auth=HTTPBasicAuth(client_id, client_secret) - ) + logger.info(f"Requesting Zoom OAuth access token for account_id={account_id}") + try: + response = requests.post( + token_url, auth=HTTPBasicAuth(client_id, client_secret) + ) + except requests.exceptions.RequestException as e: + logger.error( + f"Request failed while getting access token for account_id={account_id}: {e}" + ) + return None + if response.status_code == 200: access_token = response.json().get("access_token") + logger.info("Zoom OAuth access token obtained successfully") return access_token else: - raise Exception( - f"Failed to get access token: {response.status_code} {response.text}" + logger.error( + f"Failed to get access token for account_id={account_id}: " + f"{response.status_code} {response.text}" ) + return None def get_participants_qos(self, meeting_id, access_token, test_type="past"): url = f"https://api.zoom.us/v2/metrics/meetings/{meeting_id}/participants/qos" @@ -1467,6 +1487,11 @@ def get_live_data(self): token = self.get_access_token( self.account_id, self.client_id, self.client_secret ) + if token is None: + logger.warning( + "Unable to obtain Zoom access token; skipping this live data fetch" + ) + return self._set_raw_zoom_stats( self.get_participants_qos(self.remote_login_url, token, "live") ) @@ -1492,6 +1517,9 @@ def get_final_qos_data(self): token = self.get_access_token( self.account_id, self.client_id, self.client_secret ) + if token is None: + logger.error("Unable to obtain Zoom access token. Aborting QoS data fetch.") + return # Zoom QoS data is typically available ~20 seconds after meeting end. # We wait 150 seconds to be safe and simplify the logic. From 790783311502986fd925d7a97463939a1ba125db Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Tue, 14 Jul 2026 17:24:18 +0530 Subject: [PATCH 3/6] lf_interop_zoom.py: abort if the LANforge port lookup itself fails change_port_to_ip() wrapped the /port/{shelf}/{resource}/{port} lookup and the interface/ip key access in a single try/except, so an unreachable LANforge manager was indistinguishable from "this just isn't an ethernet port" - both fell through to the same warning and a silent fallback to the original upstream_port value. Check the raw response for None first and abort immediately with a clear error in that case; keep the existing warn-and-fallback behavior only for the genuine non-ethernet-port case (response present, but missing the interface/ip keys). Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index eb3223002..4a4726022 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -2620,8 +2620,14 @@ def change_port_to_ip(self, upstream_port): if upstream_port.count('.') != 3: target_port_list = self.name_to_eid(upstream_port) shelf, resource, port, _ = target_port_list + response = self.json_get(f'/port/{shelf}/{resource}/{port}?fields=ip') + if response is None: + logging.error( + f"GET /port/{shelf}/{resource}/{port} returned no response from LANforge. Aborting test." + ) + exit(1) try: - target_port_ip = self.json_get(f'/port/{shelf}/{resource}/{port}?fields=ip')['interface']['ip'] + target_port_ip = response['interface']['ip'] upstream_port = target_port_ip except Exception as e: logging.warning(f'The upstream port is not an ethernet port. Proceeding with the given upstream_port {upstream_port}. Exception: {e}') From 461f3bd30b1e51a1bf34b4d28a0077216f011774 Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Fri, 17 Jul 2026 11:46:26 +0530 Subject: [PATCH 4/6] lf_interop_zoom.py: retry LANforge API calls before aborting the test The immediate-exit-on-None guards added around /resource/all, /port/all, /adb and the port-to-IP lookup were too trigger-happy for a real testbed: a manager that's momentarily slow to respond would abort the whole test on the very first empty response. Add json_get_with_retry(), which retries a json_get() call every 5 seconds for up to 40 seconds before giving up, and use it at all five call sites instead of a single-shot check. get_resource_data() also gets a None-response guard for the first time here - it never actually had one. Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index 4a4726022..b85aabb22 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -829,6 +829,28 @@ def create_android( ) return True, created_cx, created_endp + def json_get_with_retry(self, url, wait_time=40, poll_interval=5): + """ + Calls self.json_get(url), retrying every poll_interval seconds for up + to wait_time seconds if LANforge returns no response. Aborts the test + if it still hasn't responded once wait_time has elapsed. + """ + start_time = time.time() + response = self.json_get(url) + while response is None and (time.time() - start_time) < wait_time: + logger.warning(f"GET {url} returned no response from LANforge; retrying...") + time.sleep(poll_interval) + response = self.json_get(url) + + if response is None: + logger.error( + f"GET {url} returned no response from LANforge after waiting " + f"{wait_time} seconds. Aborting test." + ) + exit(1) + + return response + def get_resource_data(self): self.ports_list = [] self.user_list = [] @@ -840,7 +862,7 @@ def get_resource_data(self): ] # Step 1: Retrieve information about all resources - response = self.json_get("/resource/all") + response = self.json_get_with_retry("/resource/all") # Step 2: Match user-specified resources with available resources sequentially if self.user_resources: @@ -878,10 +900,7 @@ def get_ports_data(self): self.ssid_list = [] # Step 3: Retrieve port information - response_port = self.json_get("/port/all") - if response_port is None: - logger.error("GET /port/all returned no response from LANforge. Aborting test.") - exit(1) + response_port = self.json_get_with_retry("/port/all") # Step 4: Match ports associated with retrieved resources in the order of ports_list for port_entry in self.ports_list: @@ -926,10 +945,7 @@ def get_ports_data(self): self.wifi_interface_list = [item.split(".")[2] for item in self.real_sta_list] def get_interop_data(self): - interop_data = self.json_get("/adb") - if interop_data is None: - logger.error("GET /adb returned no response from LANforge. Aborting test.") - exit(1) + interop_data = self.json_get_with_retry("/adb") interop_mobile_data = interop_data.get("devices", {}) self.serial_list = [] self.lanforge_port_list = [] @@ -1251,10 +1267,7 @@ def select_real_devices(self, real_device_obj, real_sta_list=None): if real_sta_list is None: self.real_sta_list, _, _ = real_device_obj.query_user() else: - interface_data = self.json_get("/port/all") - if interface_data is None: - logger.error("GET /port/all returned no response from LANforge. Aborting test.") - exit(1) + interface_data = self.json_get_with_retry("/port/all") interfaces = interface_data["interfaces"] final_device_list = [] # Initialize the list @@ -2620,12 +2633,7 @@ def change_port_to_ip(self, upstream_port): if upstream_port.count('.') != 3: target_port_list = self.name_to_eid(upstream_port) shelf, resource, port, _ = target_port_list - response = self.json_get(f'/port/{shelf}/{resource}/{port}?fields=ip') - if response is None: - logging.error( - f"GET /port/{shelf}/{resource}/{port} returned no response from LANforge. Aborting test." - ) - exit(1) + response = self.json_get_with_retry(f'/port/{shelf}/{resource}/{port}?fields=ip') try: target_port_ip = response['interface']['ip'] upstream_port = target_port_ip From f3504fec606651f35ad874fe044eb790b8221bb0 Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Fri, 17 Jul 2026 13:33:23 +0530 Subject: [PATCH 5/6] lf_interop_zoom.py: guard dict-key access after a successful API response json_get_with_retry() only guarantees a non-None response - it says nothing about whether that response actually has the keys these functions assume. get_resource_data(), get_ports_data() and get_interop_data() all did direct bracket access into the parsed JSON (resource_values["eid"], port_data["mac"], interop_data["devices"], etc.) with no protection, so a reshaped or partial LANforge/adb response would crash with a bare KeyError deep in the function instead of a clear diagnostic. Wrap each function's parsing logic in try/except: a KeyError branch logs the specific missing key plus the raw response received (pretty- printed via json.dumps), and a generic Exception branch logs with exc_info=True for anything else unanticipated (e.g. the IndexError possible in get_interop_data() from a malformed device identifier string). Both branches exit(1), consistent with the existing fail-fast behavior elsewhere in this file. Also switched get_interop_data()'s dict lookups from .get() to bracket access so a malformed /adb response is actually caught here instead of silently degrading into per-user "not found" placeholders. Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 225 +++++++++++------- 1 file changed, 135 insertions(+), 90 deletions(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index b85aabb22..2b1e7d6b7 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -866,31 +866,46 @@ def get_resource_data(self): # Step 2: Match user-specified resources with available resources sequentially if self.user_resources: - resources = response.get("resources", []) - lf_resources_dict = {} - for element in resources: - lf_resources_dict.update(element) - - for user_resource in self.user_resources: - if user_resource in lf_resources_dict: - resource_values = lf_resources_dict[user_resource] - eid = resource_values["eid"] - resource_ip = resource_values["ctrl-ip"] - hostname = resource_values["hostname"] - user = resource_values["user"] - - self.device_names.append(hostname) - self.ports_list.append({"eid": eid, "ctrl-ip": resource_ip}) - self.user_list.append(user) - else: - logger.error( - f"Resource {user_resource} not found in LANforge response. Aborting test." - ) - logger.info( - "LANforge resources fetched from /resource/all:\n%s", - json.dumps(lf_resources_dict, indent=2, default=str), - ) - exit(1) + try: + resources = response["resources"] + lf_resources_dict = {} + for element in resources: + lf_resources_dict.update(element) + + logger.debug( + "LANforge resources fetched from /resource/all:\n%s", + json.dumps(lf_resources_dict, indent=2, default=str), + ) + + for user_resource in self.user_resources: + if user_resource in lf_resources_dict: + resource_values = lf_resources_dict[user_resource] + eid = resource_values["eid"] + resource_ip = resource_values["ctrl-ip"] + hostname = resource_values["hostname"] + user = resource_values["user"] + + self.device_names.append(hostname) + self.ports_list.append({"eid": eid, "ctrl-ip": resource_ip}) + self.user_list.append(user) + else: + logger.error( + f"Resource {user_resource} not found in LANforge response. Aborting test." + ) + exit(1) + except KeyError as e: + logger.error( + f"/resource/all response is not in the expected format, missing key {e}. " + "Data received:\n%s", + json.dumps(response, indent=2, default=str), + ) + exit(1) + except Exception as e: + logger.error( + f"Unexpected error while parsing /resource/all response: {e}", + exc_info=True, + ) + exit(1) def get_ports_data(self): self.gen_ports_list = [] @@ -903,86 +918,116 @@ def get_ports_data(self): response_port = self.json_get_with_retry("/port/all") # Step 4: Match ports associated with retrieved resources in the order of ports_list - for port_entry in self.ports_list: - # Extract the eid and ctrl-ip from the current ports_list entry - expected_eid = port_entry["eid"] - - # Iterate over the port interfaces to find a matching port - for interface in response_port["interfaces"]: - for port, _port_data in interface.items(): - # Extract the first two segments of the port identifier to match with expected_eid - result = ".".join(port.split(".")[:2]) - - # Check if the result matches the current expected eid from ports_list - if result == expected_eid: - self.gen_ports_list.append(port.split(".")[-1]) - break - else: - continue - break + try: + for port_entry in self.ports_list: + # Extract the eid and ctrl-ip from the current ports_list entry + expected_eid = port_entry["eid"] + + # Iterate over the port interfaces to find a matching port + for interface in response_port["interfaces"]: + for port, _port_data in interface.items(): + # Extract the first two segments of the port identifier to match with expected_eid + result = ".".join(port.split(".")[:2]) + + # Check if the result matches the current expected eid from ports_list + if result == expected_eid: + self.gen_ports_list.append(port.split(".")[-1]) + break + else: + continue + break - for port_entry in self.ports_list: - # Extract the eid and ctrl-ip from the current ports_list entry - expected_eid = port_entry["eid"] + for port_entry in self.ports_list: + # Extract the eid and ctrl-ip from the current ports_list entry + expected_eid = port_entry["eid"] - # Iterate over the port interfaces to find a matching port - for interface in response_port["interfaces"]: - for port, port_data in interface.items(): - # Extract the first two segments of the port identifier to match with expected_eid - result = ".".join(port.split(".")[:2]) + # Iterate over the port interfaces to find a matching port + for interface in response_port["interfaces"]: + for port, port_data in interface.items(): + # Extract the first two segments of the port identifier to match with expected_eid + result = ".".join(port.split(".")[:2]) - # Check if the result matches the current expected eid from ports_list - if result == expected_eid and port_data["parent dev"] == "wiphy0": - self.mac_list.append(port_data["mac"]) - self.rssi_list.append(port_data["signal"]) - self.link_rate_list.append(port_data["rx-rate"]) - self.ssid_list.append(port_data["ssid"]) + # Check if the result matches the current expected eid from ports_list + if result == expected_eid and port_data["parent dev"] == "wiphy0": + self.mac_list.append(port_data["mac"]) + self.rssi_list.append(port_data["signal"]) + self.link_rate_list.append(port_data["rx-rate"]) + self.ssid_list.append(port_data["ssid"]) - break - else: - continue - break + break + else: + continue + break + except KeyError as e: + logger.error( + "/port/all response is not in the expected format, missing key %s. " + "Data received:\n%s", + e, + json.dumps(response_port, indent=2, default=str), + ) + exit(1) + except Exception as e: + logger.error( + f"Unexpected error while parsing /port/all response: {e}", + exc_info=True, + ) + exit(1) self.wifi_interface_list = [item.split(".")[2] for item in self.real_sta_list] def get_interop_data(self): interop_data = self.json_get_with_retry("/adb") - interop_mobile_data = interop_data.get("devices", {}) self.serial_list = [] self.lanforge_port_list = [] - for user in self.user_list: - if user == "": - self.serial_list.append("") - self.lanforge_port_list.append("") - else: - user_found = False - # 1. Handle Single Device (Flat Dictionary) - if isinstance(interop_mobile_data, dict): - if interop_mobile_data.get("user-name") == user: - # Extract details from 'name' (e.g., '1.1.3200f8664a91a5e9') - full_name = interop_mobile_data.get("name") - if full_name and full_name.count(".") >= 2: - resource = full_name.split(".")[1] - serial_no = full_name.split(".")[2] - self.serial_list.append(serial_no) - self.lanforge_port_list.append(f"1.{resource}.eth0") - user_found = True + try: + interop_mobile_data = interop_data["devices"] + for user in self.user_list: + if user == "": + self.serial_list.append("") + self.lanforge_port_list.append("") else: - for mobile_device in interop_mobile_data: - for serial, device_data in mobile_device.items(): - if device_data.get("user-name") == user: - resource = serial.split(".")[1] - serial_no = serial.split(".")[2] + user_found = False + # 1. Handle Single Device (Flat Dictionary) + if isinstance(interop_mobile_data, dict): + if interop_mobile_data["user-name"] == user: + # Extract details from 'name' (e.g., '1.1.3200f8664a91a5e9') + full_name = interop_mobile_data["name"] + if full_name and full_name.count(".") >= 2: + resource = full_name.split(".")[1] + serial_no = full_name.split(".")[2] self.serial_list.append(serial_no) - lanforge_port = f"1.{resource}.eth0" - self.lanforge_port_list.append(lanforge_port) + self.lanforge_port_list.append(f"1.{resource}.eth0") user_found = True + else: + for mobile_device in interop_mobile_data: + for serial, device_data in mobile_device.items(): + if device_data["user-name"] == user: + resource = serial.split(".")[1] + serial_no = serial.split(".")[2] + self.serial_list.append(serial_no) + lanforge_port = f"1.{resource}.eth0" + self.lanforge_port_list.append(lanforge_port) + user_found = True + break + if user_found: break - if user_found: - break - if not user_found: - self.serial_list.append("") - self.lanforge_port_list.append("") + if not user_found: + self.serial_list.append("") + self.lanforge_port_list.append("") + except KeyError as e: + logger.error( + "/adb response is not in the expected format, missing key %s. " + "Data received:\n%s", + e, + json.dumps(interop_data, indent=2, default=str), + ) + exit(1) + except Exception as e: + logger.error( + f"Unexpected error while parsing /adb response: {e}", + exc_info=True, + ) + exit(1) logger.debug(f"Checking serial list {self.serial_list}") From 4692da7e030160b840db1d38564dfc2558d56188 Mon Sep 17 00:00:00 2001 From: Narayana-CT Date: Fri, 17 Jul 2026 15:55:57 +0530 Subject: [PATCH 6/6] lf_interop_zoom.py: add generic endpoint status-change monitoring While a test runs, there was no visibility into individual generic endpoint state transitions - only the aggregate check_gen_cx() used to decide when the test could stop. Add monitor_endpoint_status_changes(), called once per iteration of the main test loop in run(): it polls every created endpoint and, only the first time an endpoint's status actually changes, logs it and appends a (timestamp, endpoint_name, status) row to endpoint_status_changes.csv - repeated polls of an unchanged status are silent. If every created endpoint comes back missing in a given poll (e.g. LANforge briefly unreachable), it retries every 5 seconds for up to 40 seconds before giving up; if not even one endpoint has responded by then, it logs and aborts the test, consistent with the fail-fast pattern used elsewhere in this file. Wire the new CSV into all three report-generation paths (generate_report(), generate_report_from_api(), and generate_report_from_data() via _move_report_files()) so it ends up in the report directory alongside the other per-client CSVs regardless of which path a given run takes. Verified CLI: python3 lf_interop_zoom.py --duration 2 --lanforge_ip "192.168.207.75" --signin_email "demo@gmail.com" --signin_passwd 'Demo123' --participants 1 --audio --video --upstream_port 192.168.200.170 --api_stats_collection Signed-off-by: Narayana-CT --- .../zoom_automation/lf_interop_zoom.py | 81 ++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py index 2b1e7d6b7..c81272d95 100644 --- a/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py +++ b/py-scripts/real_application_tests/zoom_automation/lf_interop_zoom.py @@ -210,6 +210,7 @@ def __init__( self.generic_endps_profile = self.new_generic_endp_profile() self.generic_endps_profile.name_prefix = "zoom" self.generic_endps_profile.type = "zoom" + self.endpoint_last_status = {} self.data_store = {} self.header = [ "timestamp", @@ -727,6 +728,71 @@ def check_gen_cx(self): logger.error(f"Error in check_gen_cx function {e}", exc_info=True) logger.info(f"generic endpoint data {generic_endpoint}") + def monitor_endpoint_status_changes(self, wait_time=40, poll_interval=5): + """ + Checks the current status of every generic endpoint and, only the + first time an endpoint's status changes, logs a message and appends + a row (timestamp, endpoint_name, status) to + endpoint_status_changes.csv. Repeated polls of an unchanged status + are not logged or written again. + + If every created endpoint is missing from the response, retries + every poll_interval seconds for up to wait_time seconds. Aborts the + test if still none of the created endpoints have responded once + wait_time has elapsed. + """ + csv_file = os.path.join(self.path, "endpoint_status_changes.csv") + created_endp = self.generic_endps_profile.created_endp + + start_time = time.time() + endpoint_data = {} + while True: + for gen_endp in created_endp: + generic_endpoint = self.json_get(f"/generic/{gen_endp}") + if generic_endpoint and "endpoint" in generic_endpoint: + endpoint_data[gen_endp] = generic_endpoint + + if endpoint_data or (time.time() - start_time) >= wait_time: + break + + logger.warning( + "No data received for any of the created endpoints; retrying..." + ) + time.sleep(poll_interval) + + if not endpoint_data: + logger.error( + f"No data received for any of the created endpoints after waiting " + f"{wait_time} seconds. Aborting test." + ) + exit(1) + + for gen_endp, generic_endpoint in endpoint_data.items(): + current_status = generic_endpoint["endpoint"].get("status", "") + previous_status = self.endpoint_last_status.get(gen_endp) + + if current_status == previous_status: + continue + + logger.info( + f"Endpoint {gen_endp} status changed to:{current_status}" + ) + + file_exists = os.path.isfile(csv_file) and os.path.getsize(csv_file) > 0 + with open(csv_file, mode="a", newline="") as f: + writer = csv.writer(f) + if not file_exists: + writer.writerow(["timestamp", "endpoint_name", "status"]) + writer.writerow( + [ + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + gen_endp, + current_status, + ] + ) + + self.endpoint_last_status[gen_endp] = current_status + def wait_for_flask(self, url="http://127.0.0.1:5000/get_latest_stats", timeout=10): """Wait until the Flask server is up, but exit if it takes longer than `timeout` seconds.""" start_time = time.time() # Record the start time @@ -1275,6 +1341,7 @@ def run(self): self.wait_for_host_ready() self.create_participants() self.wait_for_test_start() + self.monitor_endpoint_status_changes() logger.info("Monitoring the Test") time.sleep(5) if self.do_robo: @@ -1463,7 +1530,6 @@ def get_signal_and_channel_data_dict(self): def get_access_token(self, account_id, client_id, client_secret): token_url = f"https://zoom.us/oauth/token?grant_type=account_credentials&account_id={account_id}" - logger.info(f"Requesting Zoom OAuth access token for account_id={account_id}") try: response = requests.post( token_url, auth=HTTPBasicAuth(client_id, client_secret) @@ -1476,7 +1542,6 @@ def get_access_token(self, account_id, client_id, client_secret): if response.status_code == 200: access_token = response.json().get("access_token") - logger.info("Zoom OAuth access token obtained successfully") return access_token else: logger.error( @@ -2653,6 +2718,10 @@ def generate_report(self): file_to_move_path = os.path.join(self.path, f'{client}.csv') self.move_files(file_to_move_path, report_path_date_time) + self.move_files( + os.path.join(self.path, "endpoint_status_changes.csv"), report_path_date_time + ) + def change_port_to_ip(self, upstream_port): """ Convert a given port name to its corresponding IP address if it's not already an IP. @@ -3776,6 +3845,9 @@ def generate_report_from_api(self): ), self.report_path_date_time, ) + self.move_files( + os.path.join(self.path, "endpoint_status_changes.csv"), self.report_path_date_time + ) def generate_report_from_data(self): """ @@ -4040,6 +4112,11 @@ def _move_report_files(self, report_path_date_time): for f in glob.glob(pattern): self.move_files(f, report_path_date_time) + # 3. Move the endpoint status change log + self.move_files( + os.path.join(self.path, "endpoint_status_changes.csv"), report_path_date_time + ) + def stop_webui(self): """ Updates the running_status.json file to mark the test as Completed.