-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathmain.py
More file actions
126 lines (98 loc) · 4.46 KB
/
main.py
File metadata and controls
126 lines (98 loc) · 4.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
import os
import time
import signal
import logging
import threading
from datetime import datetime
from pathlib import Path
import docker
import filter as log_filter
import analyzer as log_analyzer
# --- Configuration ---
CONTAINER_NAMES = os.environ.get("CONTAINER_NAMES", "flask_open,java_springv1").split(",")
LOG_DIR = Path(os.environ.get("LOG_DIR", "/app/logs"))
POLL_INTERVAL = int(os.environ.get("POLL_INTERVAL", "5")) # seconds between container discovery checks
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
log = logging.getLogger(__name__)
shutdown_event = threading.Event()
def signal_handler(sig, frame):
log.info("Shutdown signal received, stopping log capture...")
shutdown_event.set()
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
def log_file_for(container_name: str) -> Path:
"""Return the path to the log file for a given container name."""
safe_name = container_name.replace("/", "_").lstrip("_")
return LOG_DIR / f"{safe_name}.log"
def stream_container_logs(container_name: str):
"""Stream logs from a single container and write them to a file."""
client = docker.from_env()
log_path = log_file_for(container_name)
log_path.parent.mkdir(parents=True, exist_ok=True)
log.info(f"Starting log capture for container: {container_name} -> {log_path}")
while not shutdown_event.is_set():
try:
container = client.containers.get(container_name)
with open(log_path, "a") as f:
# Stream logs since the last captured line; tail=0 means only new lines
for line in container.logs(stream=True, follow=True, timestamps=True):
if shutdown_event.is_set():
break
decoded = line.decode("utf-8", errors="replace").rstrip("\n")
f.write(decoded + "\n")
f.flush()
except docker.errors.NotFound:
log.warning(f"Container '{container_name}' not found, retrying in {POLL_INTERVAL}s...")
shutdown_event.wait(POLL_INTERVAL)
except docker.errors.APIError as e:
log.error(f"Docker API error for '{container_name}': {e}, retrying in {POLL_INTERVAL}s...")
shutdown_event.wait(POLL_INTERVAL)
except Exception as e:
log.error(f"Unexpected error for '{container_name}': {e}, retrying in {POLL_INTERVAL}s...")
shutdown_event.wait(POLL_INTERVAL)
log.info(f"Stopped log capture for container: {container_name}")
def main():
LOG_DIR.mkdir(parents=True, exist_ok=True)
log.info(f"Log capture service starting")
log.info(f"Watching containers: {CONTAINER_NAMES}")
log.info(f"Log output directory: {LOG_DIR}")
threads = []
for name in CONTAINER_NAMES:
name = name.strip()
if not name:
continue
t = threading.Thread(target=stream_container_logs, args=(name,), name=f"capture-{name}", daemon=True)
t.start()
threads.append(t)
# Share our shutdown event with the filter module
log_filter.shutdown_event = shutdown_event
# Start filter stage — watches LOG_DIR for new *.log files and filters them
filter_thread = threading.Thread(target=log_filter.watch_for_new_logs, name="filter", daemon=True)
filter_thread.start()
threads.append(filter_thread)
# Start analyzer stage — watches logs/important/ and emits alerts
log_analyzer.shutdown_event = shutdown_event
analyzer_thread = threading.Thread(target=log_analyzer.watch_important_dir, name="analyzer", daemon=True)
analyzer_thread.start()
threads.append(analyzer_thread)
# Start API server — accepts snapshot trigger requests from Flask backend
import api_server
api_server.shutdown_event = shutdown_event
api_thread = threading.Thread(target=api_server.start_api_server, name="api-server", daemon=True)
api_thread.start()
threads.append(api_thread)
# Keep main thread alive until shutdown
while not shutdown_event.is_set():
# Log a heartbeat every minute so operators know the service is running
shutdown_event.wait(60)
if not shutdown_event.is_set():
active = [t.name for t in threads if t.is_alive()]
log.info(f"Heartbeat — active capture threads: {active}")
for t in threads:
t.join(timeout=10)
log.info("Log capture service stopped.")
if __name__ == "__main__":
main()