Detection of zombi postmaster is added (NodeStatus.Zombie)

When we kill vanila-postmaster (10-18) on remote machine it becomes 'zombie'.

pg_ctl says that node is started, by by fact it is not so.

Now PostgresNode::status() check a postmaster process status (via /proc/<pid>/stat) and if 'Z' (zombie) is detected, this method returns the a status: NodeStatus.Zombie.

Just for info: PG ENT does not have a such problem.

Co-authored-by: Mark G. <mark@google.com>

Author: dmitry-lipetsk

src/enums.py

@@ -29,7 +29,7 @@ class NodeStatus(IntEnum):
     Status of a PostgresNode
     """
 
-    Running, Stopped, Uninitialized = range(3)
+    Running, Stopped, Uninitialized, Zombie = range(4)
 
     # for Python 3.x
     def __bool__(self):

src/impl/platforms/internal_platform_utils.py

@@ -60,3 +60,12 @@ def FindPostmaster(
         assert type(bin_dir) is str
         assert type(data_dir) is str
         raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.")
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+        raise NotImplementedError("InternalPlatformUtils::ProcessIsZombi_soft_ver is not implemented.")

src/impl/platforms/linux/internal_platform_utils.py

@@ -8,6 +8,7 @@
 
 import re
 import shlex
+import typing
 
 
 class InternalPlatformUtils(base.InternalPlatformUtils):
@@ -118,3 +119,53 @@ def is_space_or_tab(ch) -> bool:
         assert type(pid) is int
 
         return __class__.FindPostmasterResult.create_ok(pid)
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+
+        proc_stat_file = os_ops.build_path("/proc", str(pid), "stat")
+
+        if not os_ops.path_exists(proc_stat_file):
+            return False
+
+        result: typing.Optional[bool] = None
+
+        try:
+            # Read one line from /proc/PID/stat
+            stat_content = os_ops.read_binary(proc_stat_file, 0).decode("utf-8", errors="ignore")
+
+            # We look for the closing parenthesis of the process name to ensure that
+            # we start from it and not depend on spaces inside the parentheses!
+            r_paren_idx = stat_content.rfind(")")
+
+            if r_paren_idx == -1:
+                pass
+            elif len(stat_content) <= r_paren_idx + 2:
+                pass
+            else:
+                # The status goes exactly one space after the closing bracket
+                assert (r_paren_idx + 2) < len(stat_content)
+                proc_status = stat_content[r_paren_idx + 2]
+                result = proc_status == "Z"
+        except Exception as e:
+            # If the file disappeared right during reading, it means the process is completely erased
+            if __class__._is_file_not_found_exception(e):
+                result = False
+
+        return result
+
+    @staticmethod
+    def _is_file_not_found_exception(e: Exception) -> bool:
+        if isinstance(e, FileNotFoundError):
+            return True
+
+        if isinstance(e, ExecUtilException):
+            if e.exit_code == 2:
+                return True
+
+        return False

src/impl/platforms/win32/internal_platform_utils.py

@@ -2,6 +2,7 @@
 
 from .. import internal_platform_utils as base
 from testgres.operations.os_ops import OsOperations
+import typing
 
 
 class InternalPlatformUtils(base.InternalPlatformUtils):
@@ -15,3 +16,12 @@ def FindPostmaster(
         assert type(bin_dir) is str
         assert type(data_dir) is str
         return __class__.FindPostmasterResult.create_not_implemented()
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+        return None

src/utils.py

@@ -411,7 +411,23 @@ def get_pg_node_state(
     attempt = 0
     sleep_time = C_SLEEP_TIME1
 
-    platform_utils: typing.Optional[internal_platform_utils_factory.InternalPlatformUtils] = None
+    class tagPlaformUtilsProvider:
+        T_PLATFORM_UTILS = internal_platform_utils_factory.InternalPlatformUtils
+
+        _platform_utils: typing.Optional[T_PLATFORM_UTILS] = None
+
+        def __init__(self):
+            self._platform_utils = None
+
+        def get(self) -> T_PLATFORM_UTILS:
+            if self._platform_utils is None:
+                self._platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops)
+                assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS)
+
+            assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS)
+            return self._platform_utils
+
+    platform_utils_provider = tagPlaformUtilsProvider()
 
     while True:
         assert type(attempt) is int
@@ -513,6 +529,13 @@ def get_pg_node_state(
 
             assert pid != 0
 
+            # ----------------- detect zombie
+            if platform_utils_provider.get().ProcessIsZombi_soft_check(os_ops, pid) is True:
+                internal_utils.send_log_debug("Postmaster process {} is a zombie.".format(
+                    pid,
+                ))
+                return PostgresNodeState(NodeStatus.Zombie, pid)
+
             # -----------------
             return PostgresNodeState(NodeStatus.Running, pid)
 
@@ -543,14 +566,8 @@ def get_pg_node_state(
                     bin_dir,
                 ))
 
-            if platform_utils is None:
-                platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops)
-                assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils)
-
-            assert isinstance(platform_utils, internal_platform_utils_factory.InternalPlatformUtils)
-
             try:
-                find_postmaster_r = platform_utils.FindPostmaster(
+                find_postmaster_r = platform_utils_provider.get().FindPostmaster(
                     os_ops,
                     bin_dir,
                     data_dir,

tests/test_testgres_common.py

@@ -687,7 +687,9 @@ def test_kill__ok(
     ):
         assert isinstance(node_svc, PostgresNodeService)
 
-        with __class__.helper__get_node(node_svc) as node:
+        node = __class__.helper__get_node(node_svc)
+
+        try:
             assert isinstance(node, PostgresNode)
             assert (node.pid == 0)
             assert (node.status() == NodeStatus.Uninitialized)
@@ -696,6 +698,9 @@ def test_kill__ok(
             assert not node.is_started
             node.slow_start()
             assert node.is_started
+
+            assert node.status() == NodeStatus.Running
+
             node.kill()
             assert not node.is_started
 
@@ -717,8 +722,19 @@ def test_kill__ok(
                 if s == NodeStatus.Running:
                     continue
 
-                assert s == NodeStatus.Stopped
+                if s == NodeStatus.Stopped:
+                    logging.info("Node stopped")
+                    break
+
+                if s == NodeStatus.Zombie:
+                    logging.info("Node is zombie")
+                    break
+
+                logging.error("Node has unknown status: {}.".format(s.name))
                 break
+        finally:
+            if node.is_started:
+                node.stop()
         return
 
     def test_kill_backgroud_writer__ok(