From ff55b87544266e51059fe91eab3cae5be391b63d Mon Sep 17 00:00:00 2001 From: jmvermeulen Date: Mon, 16 Feb 2026 19:22:26 +0100 Subject: [PATCH 1/3] Optimize MP4 export: frame batching, PTS advance, ultrafast preset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reduce FPS 30→10 for forensic captures (sufficient for review) - Batch multiple renders per frame window instead of encoding per-PDU - Advance PTS to skip idle gaps without encoding duplicate still frames - Switch to h264 ultrafast preset for faster encoding - Use faststart movflag for seekable output - Set GOP size to 5s for reliable timeline seeking --- pyrdp/convert/MP4EventHandler.py | 58 ++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/pyrdp/convert/MP4EventHandler.py b/pyrdp/convert/MP4EventHandler.py index 9a8509d1f..efe217224 100644 --- a/pyrdp/convert/MP4EventHandler.py +++ b/pyrdp/convert/MP4EventHandler.py @@ -38,27 +38,21 @@ def screen(self) -> QImage: class MP4EventHandler(RenderingEventHandler): - def __init__(self, filename: str, fps=30, progress=None): + def __init__(self, filename: str, fps=10, progress=None): """ Construct an event handler that outputs to an Mp4 file. :param filename: The output file to write to. - :param fps: The frame rate (30 recommended). + :param fps: The frame rate (10 recommended for forensic captures). :param progress: An optional callback (sig: `() -> ()`) whenever a frame is muxed. """ self.filename = filename - # The movflags puts the encoder in an MP4 Streaming Format. This has two benefits: - # - recover partial videos in case of a pyrdp-convert crash - # - reduce memory consumption (especially for long captures) - # See: https://ffmpeg.org/ffmpeg-formats.html#mov_002c-mp4_002c-ismv - self.mp4 = f = av.open(filename, 'w', options={'movflags': 'frag_keyframe+empty_moov'}) + # faststart moves the moov atom to the front for seekable playback. + self.mp4 = f = av.open(filename, 'w', options={'movflags': 'faststart'}) self.stream = f.add_stream('h264', rate=fps) - # TODO: this undocumented PyAV stream feature needs to be properly investigated - # we could probably batch the encoding of several frames and benefit from threads - # but trying this as-is lead to no gains - # (actually a degradation but that could be statistically irrelevant) - #self.stream.thread_count = 4 self.stream.pix_fmt = 'yuv420p' + self.stream.options = {'preset': 'ultrafast'} + self.stream.gop_size = fps * 5 # Keyframe every 5s for seeking self.progress = progress self.scale = False self.mouse = (0, 0) @@ -67,6 +61,10 @@ def __init__(self, filename: str, fps=30, progress=None): self.log = logging.getLogger(__name__) self.log.info('Begin MP4 export to %s: %d FPS', filename, fps) self.timestamp = self.prevTimestamp = None + # PTS counter in stream time_base units for correct playback timing + self.pts = 0 + # Track whether the surface has changed since the last encoded frame + self.dirty = False super().__init__(MP4Image()) @@ -81,17 +79,33 @@ def onPDUReceived(self, pdu: PlayerPDU): self.timestamp = ts if self.prevTimestamp is None: - dt = self.delta - else: - dt = self.timestamp - self.prevTimestamp # ms + # First PDU: encode if surface was rendered + if self.dirty: + self.writeFrame() + self.dirty = False + self.prevTimestamp = ts + return + + dt = self.timestamp - self.prevTimestamp # ms nframes = (dt // self.delta) + if nframes > 0: - for _ in range(nframes): + # Frame boundary crossed. Encode one frame if surface changed, + # then advance PTS to cover any remaining idle gap. + if self.dirty: self.writeFrame() + self.dirty = False + nframes -= 1 # One frame was just encoded + # Skip remaining frames (player holds last frame) + self.pts += nframes self.prevTimestamp = ts - self.log.debug('Rendered %d still frame(s)', nframes) def cleanup(self): + # Flush any pending dirty frame + if self.dirty: + self.writeFrame() + self.dirty = False + # Add one second worth of padding so that the video doesn't end too abruptly. for _ in range(self.fps): self.writeFrame() @@ -126,9 +140,9 @@ def onCapabilities(self, caps): super().onCapabilities(caps) def onFinishRender(self): - # When the screen is updated, always write a frame. - self.prevTimestamp = self.timestamp - self.writeFrame() + # Mark surface as changed. The frame will be encoded at the next + # frame boundary in onPDUReceived, batching multiple renders. + self.dirty = True def writeFrame(self): w = self.stream.width @@ -142,8 +156,10 @@ def writeFrame(self): p.drawEllipse(x, y, 5, 5) p.end() - # Output frame. + # Output frame with explicit PTS for correct playback timing. frame = av.VideoFrame.from_ndarray(qimage2ndarray.rgb_view(surface)) + frame.pts = self.pts + self.pts += 1 for packet in self.stream.encode(frame): if self.progress: self.progress() From 688e766e9766aacef6d5f289025702eb87f2e460 Mon Sep 17 00:00:00 2001 From: jmvermeulen Date: Mon, 16 Feb 2026 19:22:34 +0100 Subject: [PATCH 2/3] Replace QImage.scaled() with 1px padding for odd H264 dimensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H264 requires even dimensions. Previously the full surface was scaled every frame to add 1px — profiling showed this cost ~6ms per frame (11% of encode time). Now pads the output surface by 1px via drawImage instead, avoiding the expensive rescale. --- pyrdp/convert/MP4EventHandler.py | 33 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pyrdp/convert/MP4EventHandler.py b/pyrdp/convert/MP4EventHandler.py index efe217224..a833d8572 100644 --- a/pyrdp/convert/MP4EventHandler.py +++ b/pyrdp/convert/MP4EventHandler.py @@ -54,7 +54,8 @@ def __init__(self, filename: str, fps=10, progress=None): self.stream.options = {'preset': 'ultrafast'} self.stream.gop_size = fps * 5 # Keyframe every 5s for seeking self.progress = progress - self.scale = False + self.padW = 0 + self.padH = 0 self.mouse = (0, 0) self.fps = fps self.delta = 1000 // fps # ms per frame @@ -125,18 +126,15 @@ def onMousePosition(self, x, y): def onCapabilities(self, caps): bmp = caps[CapabilityType.CAPSTYPE_BITMAP] (w, h) = (bmp.desktopWidth, bmp.desktopHeight) - self.imageHandler.resize(w, h) - - if w % 2 != 0: - self.scale = True - w += 1 - if h % 2 != 0: - self.scale = True - h += 1 - self.stream.width = w - self.stream.height = h + # H264 requires even dimensions. Pad by 1px instead of scaling + # every frame (scaling 2556x929 is ~6ms per frame). + self.padW = w % 2 + self.padH = h % 2 + self.stream.width = w + self.padW + self.stream.height = h + self.padH + self.imageHandler.resize(w, h) super().onCapabilities(caps) def onFinishRender(self): @@ -147,10 +145,17 @@ def onFinishRender(self): def writeFrame(self): w = self.stream.width h = self.stream.height - surface = self.imageHandler.screen.scaled(w, h) if self.scale else self.imageHandler.screen.copy() - # Draw the mouse pointer. Render mouse clicks? - p = QPainter(surface) + if self.padW or self.padH: + # Create even-sized surface and draw screen into it (avoids full scale) + surface = QImage(w, h, QImage.Format_ARGB32_Premultiplied) + p = QPainter(surface) + p.drawImage(0, 0, self.imageHandler.screen) + else: + surface = self.imageHandler.screen.copy() + p = QPainter(surface) + + # Draw the mouse pointer. p.setBrush(QColor.fromRgb(255, 255, 0, 180)) (x, y) = self.mouse p.drawEllipse(x, y, 5, 5) From 924f6513ea4ec2dedeef1dd39e1e39bbfb4e44ab Mon Sep 17 00:00:00 2001 From: jmvermeulen Date: Mon, 16 Feb 2026 21:42:32 +0100 Subject: [PATCH 3/3] Add --idle-skip option: input-based idle detection for MP4 export Detect idle periods by absence of FAST_PATH_INPUT PDUs (keyboard/mouse) instead of screen changes. Cursor blinks and screen refreshes no longer prevent idle detection. During idle, PTS is frozen with one frame encoded every 10s for state capture. True PDU gaps are also compressed. --- pyrdp/bin/convert.py | 16 ++++++- pyrdp/convert/Converter.py | 3 +- pyrdp/convert/MP4EventHandler.py | 75 ++++++++++++++++++++++++++++++-- pyrdp/convert/PCAPConverter.py | 6 +-- pyrdp/convert/ReplayConverter.py | 2 +- pyrdp/convert/utils.py | 6 ++- 6 files changed, 95 insertions(+), 13 deletions(-) diff --git a/pyrdp/bin/convert.py b/pyrdp/bin/convert.py index bbe2685bc..5123e67f8 100755 --- a/pyrdp/bin/convert.py +++ b/pyrdp/bin/convert.py @@ -57,6 +57,14 @@ def main(): "otherwise the result is output next to the source file with the proper extension. " "However if the source of the conversion is a .pcap then this option will create a directory where all files will be stored.", ) + parser.add_argument( + "--idle-skip", + type=int, + default=0, + metavar="N", + help="Skip idle periods longer than N seconds in MP4 output (0 = disabled, default: 0)", + ) + args = parser.parse_args() @@ -83,15 +91,19 @@ def main(): else: outputPrefix = "" + handler_kwargs = {} + if args.idle_skip > 0: + handler_kwargs['idle_skip'] = args.idle_skip + if inputFile.suffix in [".pcap"]: secrets = loadSecrets(args.secrets) if args.secrets else None - converter = PCAPConverter(inputFile, outputPrefix, args.format, secrets=secrets, srcFilter=args.src, dstFilter=args.dst, listOnly=args.list_only) + converter = PCAPConverter(inputFile, outputPrefix, args.format, secrets=secrets, srcFilter=args.src, dstFilter=args.dst, listOnly=args.list_only, handler_kwargs=handler_kwargs) elif inputFile.suffix in [".pyrdp"]: if args.format == "replay": sys.stderr.write("Refusing to convert a replay file to a replay file. Choose another format.") sys.exit(1) - converter = ReplayConverter(inputFile, outputPrefix, args.format) + converter = ReplayConverter(inputFile, outputPrefix, args.format, handler_kwargs=handler_kwargs) else: sys.stderr.write("Unknown file extension. (Supported: .pcap, .pyrdp)") sys.exit(1) diff --git a/pyrdp/convert/Converter.py b/pyrdp/convert/Converter.py index 604059475..d81f4a8ac 100644 --- a/pyrdp/convert/Converter.py +++ b/pyrdp/convert/Converter.py @@ -7,10 +7,11 @@ class Converter: - def __init__(self, inputFile: Path, outputPrefix: str, format: str): + def __init__(self, inputFile: Path, outputPrefix: str, format: str, handler_kwargs: dict = None): self.inputFile = inputFile self.outputPrefix = outputPrefix self.format = format + self.handler_kwargs = handler_kwargs or {} def process(self): raise NotImplementedError("Converter.process is not implemented") diff --git a/pyrdp/convert/MP4EventHandler.py b/pyrdp/convert/MP4EventHandler.py index a833d8572..6b98ec571 100644 --- a/pyrdp/convert/MP4EventHandler.py +++ b/pyrdp/convert/MP4EventHandler.py @@ -4,7 +4,7 @@ # Licensed under the GPLv3 or later. # -from pyrdp.enum import CapabilityType +from pyrdp.enum import CapabilityType, PlayerPDUType from pyrdp.pdu import PlayerPDU from pyrdp.player.ImageHandler import ImageHandler from pyrdp.player.RenderingEventHandler import RenderingEventHandler @@ -38,13 +38,14 @@ def screen(self) -> QImage: class MP4EventHandler(RenderingEventHandler): - def __init__(self, filename: str, fps=10, progress=None): + def __init__(self, filename: str, fps=10, progress=None, idle_skip=0): """ Construct an event handler that outputs to an Mp4 file. :param filename: The output file to write to. :param fps: The frame rate (10 recommended for forensic captures). :param progress: An optional callback (sig: `() -> ()`) whenever a frame is muxed. + :param idle_skip: Seconds of inactivity before compressing idle gaps (0 = disabled). """ self.filename = filename # faststart moves the moov atom to the front for seekable playback. @@ -66,6 +67,13 @@ def __init__(self, filename: str, fps=10, progress=None): self.pts = 0 # Track whether the surface has changed since the last encoded frame self.dirty = False + # Idle skip: based on user input (keyboard/mouse), not screen changes + self.idle_skip_ms = idle_skip * 1000 if idle_skip > 0 else 0 + self.total_skipped_ms = 0 + self.lastInputTimestamp = None + self._in_idle = False + self._idle_enter_ts = None + self._last_idle_frame_ts = None super().__init__(MP4Image()) @@ -78,16 +86,56 @@ def onPDUReceived(self, pdu: PlayerPDU): ts = pdu.timestamp self.timestamp = ts + is_input = pdu.header == PlayerPDUType.FAST_PATH_INPUT + + # Track user input for idle detection + if is_input: + self.lastInputTimestamp = ts if self.prevTimestamp is None: - # First PDU: encode if surface was rendered + # First PDU: assume active at start + if self.lastInputTimestamp is None: + self.lastInputTimestamp = ts + if self.dirty: + self.writeFrame() + self.dirty = False + self.prevTimestamp = ts + return + + # Check input-idle state + input_idle_ms = ts - self.lastInputTimestamp + now_idle = self.idle_skip_ms > 0 and input_idle_ms > self.idle_skip_ms + + if now_idle and not self._in_idle: + # Entering idle: encode last dirty frame, then freeze + self._in_idle = True + self._idle_enter_ts = ts + self._last_idle_frame_ts = ts if self.dirty: self.writeFrame() self.dirty = False + + if self._in_idle and not now_idle: + # Exiting idle (input resumed) + self._in_idle = False + self.total_skipped_ms += ts - self._idle_enter_ts + self.pts += self.fps # 1s pause in output + self.prevTimestamp = ts + return + + if self._in_idle: + # During idle: encode 1 frame every 10s to capture screen state + if self.dirty and (ts - self._last_idle_frame_ts) >= 10000: + self.writeFrame() + self.dirty = False + self._last_idle_frame_ts = ts + else: + self.dirty = False self.prevTimestamp = ts return - dt = self.timestamp - self.prevTimestamp # ms + # Normal (non-idle) processing + dt = ts - self.prevTimestamp # ms nframes = (dt // self.delta) if nframes > 0: @@ -97,11 +145,23 @@ def onPDUReceived(self, pdu: PlayerPDU): self.writeFrame() self.dirty = False nframes -= 1 # One frame was just encoded + + # True gap (no PDUs at all): compress + gap_threshold = int(self.idle_skip_ms / self.delta) if self.idle_skip_ms > 0 else 0 + if gap_threshold > 0 and nframes > gap_threshold: + self.total_skipped_ms += nframes * self.delta + nframes = self.fps # Replace gap with 1s pause + # Skip remaining frames (player holds last frame) self.pts += nframes self.prevTimestamp = ts def cleanup(self): + # Close out idle state if capture ends while idle + if self._in_idle and self._idle_enter_ts and self.timestamp: + self.total_skipped_ms += self.timestamp - self._idle_enter_ts + self._in_idle = False + # Flush any pending dirty frame if self.dirty: self.writeFrame() @@ -116,6 +176,13 @@ def cleanup(self): if self.progress: self.progress() self.mp4.mux(pkt) + + if self.total_skipped_ms > 0: + skipped_s = self.total_skipped_ms / 1000 + m, s = divmod(int(skipped_s), 60) + h, m = divmod(m, 60) + self.log.info('Total idle time skipped: %dh %dm %ds (%.1fs)', h, m, s, skipped_s) + self.log.info('Export completed.') self.mp4.close() diff --git a/pyrdp/convert/PCAPConverter.py b/pyrdp/convert/PCAPConverter.py index 7f87ad130..90a8999fc 100644 --- a/pyrdp/convert/PCAPConverter.py +++ b/pyrdp/convert/PCAPConverter.py @@ -24,8 +24,8 @@ class PCAPConverter(Converter): SESSIONID_FORMAT = "{timestamp}_{src}-{dst}" - def __init__(self, inputFile: Path, outputPrefix: str, format: str, secrets: Dict = None, srcFilter = None, dstFilter = None, listOnly = False): - super().__init__(inputFile, outputPrefix, format) + def __init__(self, inputFile: Path, outputPrefix: str, format: str, secrets: Dict = None, srcFilter = None, dstFilter = None, listOnly = False, handler_kwargs: dict = None): + super().__init__(inputFile, outputPrefix, format, handler_kwargs=handler_kwargs) self.secrets = secrets if secrets is not None else {} self.srcFilter = srcFilter if srcFilter is not None else srcFilter self.dstFilter = dstFilter if dstFilter is not None else dstFilter @@ -106,7 +106,7 @@ def processStream(self, startTimeStamp: int, stream: PCAPStream): }) sessionID = sessionID.replace(":", "_") - handler, _ = createHandler(self.format, self.outputPrefix + sessionID) + handler, _ = createHandler(self.format, self.outputPrefix + sessionID, **self.handler_kwargs) replayer = RDPReplayer(handler, self.outputPrefix, sessionID) print(f"[*] Processing {stream.client} -> {stream.server}") diff --git a/pyrdp/convert/ReplayConverter.py b/pyrdp/convert/ReplayConverter.py index 0e66ccd6c..193c6f7c5 100644 --- a/pyrdp/convert/ReplayConverter.py +++ b/pyrdp/convert/ReplayConverter.py @@ -20,7 +20,7 @@ def process(self): print(f"[*] Converting '{self.inputFile}' to {self.format.upper()}") outputFileBase = self.outputPrefix + self.inputFile.stem - handler, outputPath = createHandler(self.format, outputFileBase) + handler, outputPath = createHandler(self.format, outputFileBase, **self.handler_kwargs) if not handler: print("The input file is already a replay file. Nothing to do.") diff --git a/pyrdp/convert/utils.py b/pyrdp/convert/utils.py index 0a75e1a86..6fbc70203 100644 --- a/pyrdp/convert/utils.py +++ b/pyrdp/convert/utils.py @@ -74,7 +74,7 @@ def extractInetAddressesFromPDUPacket(packet) -> Tuple[InetAddress, InetAddress] return (InetAddress(x.src, x.sport), InetAddress(x.dst, x.dport)) -def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, str]: +def createHandler(format: str, outputFileBase: str, progress=None, **kwargs) -> Tuple[str, str]: """ Gets the appropriate handler and returns the filename with extension. Returns None if the format is replay. @@ -87,7 +87,9 @@ def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, HandlerClass, ext = HANDLERS[format] outputFileBase += f".{ext}" - return HandlerClass(outputFileBase, progress=progress) if HandlerClass else None, outputFileBase + if HandlerClass: + return HandlerClass(outputFileBase, progress=progress, **kwargs), outputFileBase + return None, outputFileBase class ExportedPDU(Packet):