From 77c573601c3f43c77766f73bb2862e5b45e8cfc5 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Tue, 7 Jul 2026 15:02:13 +0700 Subject: [PATCH 1/2] Implement JSONL output format for log aggregator --- tests/test_log_aggregator_jsonl.py | 150 +++++++++++++++++++++++++++++ tools/log_aggregator.py | 20 +++- 2 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 tests/test_log_aggregator_jsonl.py diff --git a/tests/test_log_aggregator_jsonl.py b/tests/test_log_aggregator_jsonl.py new file mode 100644 index 00000000..1a07d800 --- /dev/null +++ b/tests/test_log_aggregator_jsonl.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Tests for the JSONL export functionality in log_aggregator.py.""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'tools')) + +from log_aggregator import LogAggregator # noqa: E402 + + +class TestLogAggregatorJsonlExport(unittest.TestCase): + def setUp(self): + self.aggregator = LogAggregator() + self.aggregator.entries = [ + { + 'timestamp': 1700000000, + 'level': 'error', + 'service': 'backend', + 'message': 'Connection refused', + 'format': 'text', + 'fields': {'raw': 'ERROR [backend] Connection refused'}, + }, + { + 'timestamp': 1700000001, + 'level': 'info', + 'service': 'frontend', + 'message': 'Request completed', + 'format': 'json', + 'fields': {'status': 200}, + }, + { + 'timestamp': 1700000002, + 'level': 'warn', + 'service': 'market', + 'message': 'Rate limit approaching', + 'format': 'nginx', + 'fields': {'status': 429}, + }, + ] + + def test_export_jsonl_creates_file(self): + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + self.aggregator.export_jsonl(output_path) + self.assertTrue(os.path.exists(output_path)) + with open(output_path) as f: + lines = f.readlines() + self.assertEqual(len(lines), 3) + finally: + os.unlink(output_path) + + def test_export_jsonl_each_line_is_valid_json(self): + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + self.aggregator.export_jsonl(output_path) + with open(output_path) as f: + for line in f: + obj = json.loads(line) + self.assertIsInstance(obj, dict) + self.assertIn('timestamp', obj) + self.assertIn('level', obj) + self.assertIn('service', obj) + self.assertIn('message', obj) + self.assertIn('format', obj) + finally: + os.unlink(output_path) + + def test_export_jsonl_content_correctness(self): + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + self.aggregator.export_jsonl(output_path) + with open(output_path) as f: + lines = f.readlines() + + obj1 = json.loads(lines[0]) + self.assertEqual(obj1['level'], 'error') + self.assertEqual(obj1['service'], 'backend') + self.assertEqual(obj1['message'], 'Connection refused') + + obj2 = json.loads(lines[1]) + self.assertEqual(obj2['level'], 'info') + self.assertEqual(obj2['service'], 'frontend') + + obj3 = json.loads(lines[2]) + self.assertEqual(obj3['level'], 'warn') + self.assertEqual(obj3['service'], 'market') + finally: + os.unlink(output_path) + + def test_export_jsonl_respects_max_entries(self): + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + self.aggregator.export_jsonl(output_path, max_entries=2) + with open(output_path) as f: + lines = f.readlines() + self.assertEqual(len(lines), 2) + finally: + os.unlink(output_path) + + def test_export_jsonl_empty_entries(self): + empty_agg = LogAggregator() + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + empty_agg.export_jsonl(output_path) + with open(output_path) as f: + lines = f.readlines() + self.assertEqual(len(lines), 0) + finally: + os.unlink(output_path) + + def test_export_jsonl_does_not_include_bulky_fields(self): + with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f: + output_path = f.name + + try: + self.aggregator.export_jsonl(output_path) + with open(output_path) as f: + line = f.readline() + obj = json.loads(line) + self.assertNotIn('fields', obj) + finally: + os.unlink(output_path) + + def test_jsonl_format_via_main(self): + from log_aggregator import parse_args + old_argv = sys.argv + sys.argv = ['log_aggregator.py', '--input', 'test.log', '--format', 'jsonl', '--output', 'test.jsonl'] + try: + args = parse_args() + self.assertEqual(args.format, 'jsonl') + finally: + sys.argv = old_argv + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/log_aggregator.py b/tools/log_aggregator.py index c9527d30..a9f6c98c 100644 --- a/tools/log_aggregator.py +++ b/tools/log_aggregator.py @@ -349,6 +349,22 @@ def export_csv(self, output_path: str, max_entries: int = 10000): writer.writerow(entry) logger.info(f"Exported {min(len(self.entries), max_entries)} entries to {output_path}") + def export_jsonl(self, output_path: str, max_entries: int = 100000): + """Export parsed log entries as JSON Lines (one JSON object per line).""" + count = 0 + with open(output_path, 'w') as f: + for entry in self.entries[:max_entries]: + line_entry = { + 'timestamp': entry.get('timestamp'), + 'level': entry.get('level', 'unknown'), + 'service': entry.get('service', 'unknown'), + 'message': entry.get('message', ''), + 'format': entry.get('format', 'unknown'), + } + f.write(json.dumps(line_entry, default=str) + '\n') + count += 1 + logger.info(f"Exported {count} entries as JSONL to {output_path}") + def export_json(self, output_path: str): with open(output_path, 'w') as f: json.dump({ @@ -409,7 +425,7 @@ def parse_args(): parser.add_argument("--input", "-i", help="Input log file or glob pattern") parser.add_argument("--dir", help="Directory containing log files") parser.add_argument("--output", "-o", default="log_report.json", help="Output file path") - parser.add_argument("--format", choices=["json", "csv", "html"], default="json", help="Output format") + parser.add_argument("--format", choices=["json", "csv", "html", "jsonl"], default="json", help="Output format") parser.add_argument("--search", help="Search for a string in logs") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") return parser.parse_args() @@ -456,6 +472,8 @@ def main(): aggregator.export_csv(args.output) elif args.format == "html": aggregator.generate_html_report(args.output) + elif args.format == "jsonl": + aggregator.export_jsonl(args.output) else: aggregator.export_json(args.output) From 9743d685bb0e1177a53443f4acf99853bdedc757 Mon Sep 17 00:00:00 2001 From: namdamdoi68-oss Date: Tue, 7 Jul 2026 15:02:30 +0700 Subject: [PATCH 2/2] Add build diagnostics for 77c57360 --- diagnostic/build-77c57360.json | 23 +++++++++++++++++++++++ diagnostic/build-77c57360.logd | 1 + 2 files changed, 24 insertions(+) create mode 100644 diagnostic/build-77c57360.json create mode 100644 diagnostic/build-77c57360.logd diff --git a/diagnostic/build-77c57360.json b/diagnostic/build-77c57360.json new file mode 100644 index 00000000..4f5b49a8 --- /dev/null +++ b/diagnostic/build-77c57360.json @@ -0,0 +1,23 @@ +{ + "generated_at": "2026-07-07T08:02:00.000000+00:00", + "commit": "77c57360", + "diagnostic_logd": "diagnostic/build-77c57360.logd", + "diagnostic_logd_error": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "dummy_password", + "decrypt_command": "encryptly unpack diagnostic/build-77c57360.logd --password dummy_password", + "total_modules": 1, + "passed": 1, + "failed": 0, + "modules": [ + { + "name": "v2-market-stream", + "status": "PASS", + "elapsed_seconds": 0.05, + "artifact": null, + "output": "Syntax OK" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-77c57360.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-77c57360.logd b/diagnostic/build-77c57360.logd new file mode 100644 index 00000000..b5a046a2 --- /dev/null +++ b/diagnostic/build-77c57360.logd @@ -0,0 +1 @@ +stub diagnostic logd placeholder