Skip to content

Commit ebedf60

Browse files
committed
feat(sdk): migrate deployment logs to logs_v4 and add pod listing
Adds get_deployment_pods, a page-based get_deployment_logs anchored on previously fetched events (before/after), and a stateful DeploymentLogSession wrapping that primitive for backfill, tailing, and cross-process resume. Signed-off-by: Honglin Cao <hocao@nvidia.com>
1 parent 17c38b8 commit ebedf60

4 files changed

Lines changed: 419 additions & 91 deletions

File tree

‎README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,19 @@ Use `python examples/sdk/get_clusters.py` and
5353
Creating the example reserves GPU capacity and may incur usage charges. It does not
5454
delete the deployment automatically.
5555

56+
### Deployment logs SDK example
57+
58+
Logs are read per pod. Discover pod names with `get_deployment_pods()` (terminated
59+
pods still within log retention are included), then read with a
60+
`deployment_log_session()`: `fetch_older()` pages toward the beginning of history and
61+
`fetch_newer()` returns only new lines, while the session keeps the merged, ordered
62+
log in `.events`. The same paging is available statelessly through
63+
`get_deployment_logs(before=..., after=...)`, anchored on events you already hold:
64+
65+
```bash
66+
python examples/sdk/get_deployment_logs.py
67+
```
68+
5669
### Un-installation
5770

5871
To uninstall `centml`, simply do:

‎centml/sdk/api.py‎

Lines changed: 122 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
from bisect import insort
12
from contextlib import contextmanager
3+
from typing import List, Optional
24

35
import platform_api_python_client
46
from platform_api_python_client import (
@@ -21,6 +23,11 @@
2123

2224
STATUS_V3_DEPLOYMENT_TYPES = {DeploymentType.INFERENCE_V3, DeploymentType.CSERVE_V3}
2325

26+
DEFAULT_LOG_PAGE_LINES = 100 # server-side default for max_lines
27+
# The server re-delivers a ~15s look-behind window on fetch-newer requests; only the
28+
# caller's events within this generous margin of the boundary can be re-delivered.
29+
LOG_DEDUP_RETENTION_MS = 300_000
30+
2431

2532
class CentMLClient:
2633
def __init__(self, api):
@@ -191,46 +198,127 @@ def get_deployment_revisions(self, deployment_id: int):
191198
deployment_id=deployment_id
192199
).results
193200

201+
def get_deployment_pods(self, deployment_id: int, revision_number: int) -> List[str]:
202+
"""List pods that have logged for a deployment revision, including terminated
203+
pods still within log retention. A fresh deployment may return an empty list."""
204+
return self._api.get_deployment_pods_deployments_pods_deployment_id_revision_number_get(
205+
deployment_id=deployment_id, revision_number=revision_number
206+
).pods
207+
208+
# pylint: disable=R0917
194209
def get_deployment_logs(
195210
self,
196211
deployment_id: int,
197212
revision_number: int,
198-
start_time: int,
199-
end_time: int,
200-
line_count: int = 100,
201-
start_from_head: bool = True,
202-
stream: bool = False,
203-
):
204-
"""Fetch logs for a deployment within a time window, handling pagination automatically.
205-
206-
start_time and end_time are Unix timestamps in milliseconds.
207-
Use get_deployment_revisions() to find the current revision number.
208-
209-
If stream=True, returns a generator that yields events as each page is fetched.
210-
If stream=False (default), returns a flat list of all events.
213+
pod: str,
214+
before: Optional[list] = None,
215+
after: Optional[list] = None,
216+
max_lines: int = DEFAULT_LOG_PAGE_LINES,
217+
) -> list:
218+
"""Fetch one page of a pod's logs, oldest-first. Use get_deployment_pods() to
219+
discover pod names and get_deployment_revisions() for the revision number.
220+
221+
before and after anchor the page to events a previous call returned for the
222+
same pod (pass your accumulated list; only the relevant boundary is used):
223+
- neither: the newest page (tail).
224+
- before=<events>: the page strictly older than the oldest of them;
225+
an empty result means the beginning of history is reached.
226+
- after=<events>: lines strictly newer than the newest of them; an empty
227+
result means nothing new yet — call again later to keep tailing. Late
228+
lines still landing near that boundary are included on top of max_lines
229+
and may sort below events you already hold (order by id if that matters).
211230
"""
231+
if before is not None and after is not None:
232+
raise ValueError("before and after are mutually exclusive")
233+
234+
fetch_newer = after is not None
235+
anchor_events = after if fetch_newer else before
236+
boundary_timestamp = None
237+
if anchor_events:
238+
timestamps = [event.timestamp for event in anchor_events]
239+
boundary_timestamp = max(timestamps) if fetch_newer else min(timestamps)
240+
241+
response = self._api.get_deployment_logs_v4_logs_deployment_id_revision_number_get(
242+
deployment_id=deployment_id,
243+
revision_number=revision_number,
244+
pod=pod,
245+
fetch_newer=fetch_newer,
246+
timestamp=boundary_timestamp,
247+
max_lines=max_lines,
248+
)
249+
if not fetch_newer or not anchor_events:
250+
return response.events
251+
252+
# fetch_newer re-delivers a look-behind window at and before the boundary
253+
# (late-arrival protection); drop the lines the caller already holds by id.
254+
cutoff = max(event.timestamp for event in anchor_events) - LOG_DEDUP_RETENTION_MS
255+
held_event_ids = {event.id for event in anchor_events if event.timestamp >= cutoff}
256+
return [event for event in response.events if event.id not in held_event_ids]
257+
258+
def deployment_log_session(
259+
self, deployment_id: int, revision_number: int, pod: str, events: Optional[list] = None
260+
) -> "DeploymentLogSession":
261+
"""Stateful reader for one pod's logs that tracks fetched pages and anchors
262+
every request itself — see DeploymentLogSession. Seed events with logs a
263+
previous session (or get_deployment_logs) returned for the same pod."""
264+
return DeploymentLogSession(self, deployment_id, revision_number, pod, events)
265+
266+
267+
class DeploymentLogSession:
268+
"""Maintains a contiguous, ordered window of one pod's logs across fetches.
269+
270+
Every fetch is anchored on the window itself, so pages can never overlap or
271+
leave gaps inside it (within log retention; an undetectable gap forms if the
272+
session idles past retention before fetching newer lines).
273+
"""
212274

213-
def _iter_events():
214-
next_page_token = None
215-
while True:
216-
response = self._api.get_deployment_logs_v3_deployments_logs_v3_deployment_id_revision_number_get(
217-
deployment_id=deployment_id,
218-
revision_number=revision_number,
219-
start_time=start_time,
220-
end_time=end_time,
221-
next_page_token=next_page_token,
222-
start_from_head=start_from_head,
223-
line_count=line_count,
224-
)
225-
yield from response.events
226-
next_page_token = response.next_page_token
227-
if not next_page_token:
228-
break
229-
230-
if stream:
231-
return _iter_events()
232-
233-
return list(_iter_events())
275+
# pylint: disable=R0917
276+
def __init__(self, client: CentMLClient, deployment_id: int, revision_number: int, pod: str, events=None):
277+
self._client = client
278+
self._deployment_id = deployment_id
279+
self._revision_number = revision_number
280+
self._pod = pod
281+
# Seeded events come from outside the session: canonicalize to unique ids in
282+
# chronological order (id order == time order at nanosecond precision).
283+
unique_events = {event.id: event for event in events or []}
284+
self._events = [unique_events[event_id] for event_id in sorted(unique_events)]
285+
286+
@property
287+
def events(self) -> list:
288+
"""Copy of the window fetched so far, oldest first. Complete from the beginning
289+
of history only once fetch_older() has returned an empty list."""
290+
return list(self._events)
291+
292+
def fetch_older(self, max_lines: int = DEFAULT_LOG_PAGE_LINES) -> list:
293+
"""Fetch the page older than the window and prepend it; on an empty session
294+
fetches the newest page (tail). Returns the page; empty list = no older
295+
lines exist (yet)."""
296+
page = self._client.get_deployment_logs(
297+
self._deployment_id, self._revision_number, self._pod, before=self._events, max_lines=max_lines
298+
)
299+
self._events[:0] = page
300+
return page
301+
302+
def fetch_newer(self, max_lines: int = DEFAULT_LOG_PAGE_LINES) -> list:
303+
"""Fetch lines newer than the window and merge them in; on an empty session
304+
fetches the newest page (tail) — to read from the beginning of history
305+
instead, loop fetch_older() until it returns an empty list. Returns only
306+
the new lines; empty list = nothing new yet, call again later to keep
307+
tailing. Rare late arrivals sort into the window below its newest lines."""
308+
if not self._events:
309+
return self.fetch_older(max_lines=max_lines)
310+
delta = self._client.get_deployment_logs(
311+
self._deployment_id, self._revision_number, self._pod, after=self._events, max_lines=max_lines
312+
)
313+
for event in delta:
314+
if event.id > self._events[-1].id:
315+
self._events.append(event)
316+
else:
317+
# A late arrival may even precede the window's oldest line (tail page
318+
# cut inside the look-behind span); the server delivers that span
319+
# completely on top of max_lines, so the window stays contiguous.
320+
insort(self._events, event, key=lambda held: held.id)
321+
return delta
234322

235323

236324
@contextmanager

‎examples/sdk/get_deployment_logs.py‎

Lines changed: 37 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,54 @@
1-
from datetime import datetime, timezone, timedelta
1+
import time
2+
from datetime import datetime, timezone
23

34
from centml.sdk.api import get_centml_client
45

56
# --- Configuration ---
67
DEPLOYMENT_ID = 1234 # Replace with your deployment ID
78
REVISION_NUMBER = 10
8-
HOURS_BACK = 1 # Fetch logs from the last N hours
9+
TAIL_SECONDS = 30 # How long to keep polling for new lines after reading history
910

1011

11-
def format_event(event: dict) -> str:
12-
timestamp_ms = (
13-
event.get("timestamp")
14-
or event.get("time")
15-
or event.get("ts")
16-
or ""
17-
)
18-
message = (
19-
event.get("message")
20-
or event.get("msg")
21-
or event.get("log")
22-
or str(event)
23-
)
24-
if timestamp_ms:
25-
ts = datetime.fromtimestamp(int(timestamp_ms) / 1000, tz=timezone.utc).isoformat()
26-
return f"[{ts}] {message}"
27-
return message
12+
def format_event(event) -> str:
13+
ts = datetime.fromtimestamp(event.timestamp / 1000, tz=timezone.utc).isoformat()
14+
return f"[{ts}] {event.message}"
2815

2916

3017
def main():
31-
stream = True
32-
end_time = int(datetime.now(timezone.utc).timestamp() * 1000)
33-
start_time = end_time - int(timedelta(hours=HOURS_BACK).total_seconds() * 1000)
34-
35-
print(f"Fetching logs for deployment {DEPLOYMENT_ID}")
36-
print(
37-
f"Time window: "
38-
f"{datetime.fromtimestamp(start_time / 1000, tz=timezone.utc).isoformat()} → "
39-
f"{datetime.fromtimestamp(end_time / 1000, tz=timezone.utc).isoformat()}"
40-
)
41-
print()
42-
4318
with get_centml_client() as cclient:
44-
if stream:
45-
# Streaming: print events as each page arrives
46-
for event in cclient.get_deployment_logs(
47-
deployment_id=DEPLOYMENT_ID,
48-
revision_number=REVISION_NUMBER,
49-
start_time=start_time,
50-
end_time=end_time,
51-
start_from_head=False,
52-
stream=stream,
53-
):
19+
# Logs are read per pod: discover the pods that have logged for this revision
20+
# (terminated pods within log retention are included).
21+
pods = cclient.get_deployment_pods(DEPLOYMENT_ID, REVISION_NUMBER)
22+
if not pods:
23+
print("No pods have logged for this revision yet.")
24+
return
25+
26+
pod = pods[0]
27+
print(f"Reading logs for deployment {DEPLOYMENT_ID} revision {REVISION_NUMBER}, pod {pod}\n")
28+
29+
# The session tracks what it has fetched and anchors every request itself.
30+
session = cclient.deployment_log_session(DEPLOYMENT_ID, REVISION_NUMBER, pod)
31+
32+
# Read the full history: newest page first, then page back to the beginning.
33+
while session.fetch_older():
34+
pass
35+
print(f"Found {len(session.events)} log entries:\n")
36+
for event in session.events:
37+
print(format_event(event))
38+
39+
# Keep tailing: each call returns only the lines the session does not hold yet.
40+
print(f"\nPolling for new lines for {TAIL_SECONDS}s...")
41+
deadline = time.monotonic() + TAIL_SECONDS
42+
while time.monotonic() < deadline:
43+
for event in session.fetch_newer():
5444
print(format_event(event))
55-
else:
56-
# Batch: collect all events then process
57-
events = cclient.get_deployment_logs(
58-
deployment_id=DEPLOYMENT_ID,
59-
revision_number=REVISION_NUMBER,
60-
start_time=start_time,
61-
end_time=end_time,
62-
start_from_head=False,
63-
)
45+
time.sleep(2)
6446

65-
if not events:
66-
print("No logs found in the given time window.")
67-
return
68-
69-
print(f"Found {len(events)} log entries:\n")
70-
for event in events:
71-
print(format_event(event))
47+
# The same paging is available statelessly via get_deployment_logs, anchored
48+
# on events you already hold — useful when you manage storage yourself:
49+
# page = cclient.get_deployment_logs(DEPLOYMENT_ID, REVISION_NUMBER, pod=pod) # tail
50+
# older = cclient.get_deployment_logs(..., pod=pod, before=page) # [] = beginning
51+
# newer = cclient.get_deployment_logs(..., pod=pod, after=page) # [] = nothing new
7252

7353

7454
if __name__ == "__main__":

0 commit comments

Comments
 (0)