功能:完善2K串流、显示恢复与输入优化

This commit is contained in:
songyiming committed 2026-11-03 13:43:50 +08:00
1 parent e47b01beec
commit 938ec97cce
95 files changed
+5788 -72

No files matched your search

+40
View File
@@ -0,0 +1,40 @@
# VDI Moonlight 交互优化补丁
本目录维护当前 RK3528 / Xorg POC 私有客户端的源码修改,不替换系统 Moonlight。
基线:Moonlight Qt v6.2.0 `de2467e433821664cdd2224aad8c89a625be1ad9`,其检出的 moonlight-common-c 子模块保持对应版本。MPP、FFmpeg 版本仍由 `tools/build-rkmpp-client.sh` 固定。
## 补丁
- `0001-vdi-input-and-latest-frame.patch` 在 Moonlight Qt 根目录应用。鼠标移动只合并队首连续事件,遇到按钮、键盘、窗口、呈现事件或其他鼠标设备立即停止;每次最多额外合并 64 个事件。绝对模式在发送按键前使用该按键事件的坐标。低延迟队列可选地把已等待的解码帧替换成新帧。
- `0002-vdi-absolute-click-barrier.patch` 在 `moonlight-common-c/moonlight-common-c` 中应用。鼠标按钮封住此前的位置批次,后续移动只能进入新批次;发送线程只锁定自己的批次,避免旧包出队时误清除新批次。所有包依旧走原可靠鼠标通道,保留 1ms 合并限流和原坐标边界修正。
- `0003-vdi-input-priority.patch` 在 Qt 根目录应用。可选地让帧就绪后的连续输入先执行,最多 32 条或 2ms,不能跨窗口/焦点/其他事件。`VDI_INPUT_PRIORITY=1` 才启用,默认关闭。另提供 `VDI_INPUT_TIMING=1` 的汇总统计,区分 SDL 输入派发年龄与主线程呈现调用耗时,不代表端到端延迟。
2026-11-03 的现场 RKMPP/EGL 使用独立 PacerRender 线程,诊断会话主线程 render 调用数为 0;因此第三补丁的新调度没有已证明的现场收益,未启用。诊断候选二进制保存为 `moonlight.input-priority-20261103`;默认客户端已恢复到本轮前 SHA256 `35de7b1a4ade583823b23b5f2e012307f6d0a3848c8eb97bd6d4de9660afa00f`。详细计划/初测限制见 `docs/mouse-optimization-plan-20261103.md`,独立光标接入设计见 `docs/independent-cursor-design-20261103.md`。
`VDI_LATEST_FRAME=1` 只在没有 VSync pacing source 的呈现队列生效。最多保留一个待呈现解码帧,丢弃数计入原 `pacerDroppedFrames`。不丢压缩包,不改变 H.264 参考链,也不提前释放正在呈现或延迟释放的上一帧。未设置或设为 `0` 时保留上游帧队列策略;kiosk 私有 wrapper 默认设为 `1`。smooth 档仍使用原 pacing 行为。
这些补丁不增加本地光标,不隐藏 Windows 光标,不改变控制协议。远程光标仍随视频显示,其延迟仍受服务器捕获/编码、网络、解码和显示影响。
## 构建与测试
在盒子上运行构建器时,将本目录的 `patches/` 一起复制过去,通过 `VDI_MOONLIGHT_PATCH_DIR` 指定目录。构建器先检查固定提交,再检查补丁:已完整应用则跳过;上下文不匹配则失败,不强行覆盖其他修改。
```bash
VDI_BUILD_STAGE=moonlight \
VDI_MOONLIGHT_PATCH_DIR="$HOME/vdi-build/rkmpp-experiment/patches" \
bash tools/build-rkmpp-client.sh
```
`tests/` 的文件复制到盒子 `$VDI_RKMPP_BUILD_ROOT/latency-tests/` 后运行:
```bash
bash tools/test-moonlight-latency.sh
VDI_TEST_SANITIZE=1 ASAN_OPTIONS=detect_leaks=1 bash tools/test-moonlight-latency.sh
```
测试直接包含实际修改后的 SDL helper、AVFrame 队列 helper 与 `InputStream.c`,并链接客户端实际使用的 native 队列/线程库。覆盖跨点击合并、键盘/呈现边界、工作量上限、触摸隔离、200 帧的积压替换、已呈现帧所有权及 1,000 轮“移动→按下→移动→释放→移动”的发送线程滞后场景。测试显式启用断言,避免上游 release 头文件的 NDEBUG 使检查失效。
新增 `test_input_priority.cpp` 验证优先调度不越过焦点/窗口/退出/解码器重置事件,保持输入 FIFO,隔离其他窗口与合成触摸,并验证数量与时间限制。普通与 ASAN 版本均通过。当前 RKMPP/EGL 没有主线程呈现事件,新调度开关不能用作该路径的性能提升宣传。
现场数据、局限与完整回退入口见 `docs/input-rendering-optimization-20261103.md`。
@@ -0,0 +1,271 @@
--- a/app/streaming/input/mouse.cpp
+++ b/app/streaming/input/mouse.cpp
@@ -1,4 +1,5 @@
#include "input.h"
+#include "vdi_mouse_batch.h"
#include <Limelight.h>
#include "SDL_compat.h"
@@ -84,6 +85,11 @@
button = BUTTON_RIGHT;
}
+ // Button coordinates are captured with this event, not a later mouse state.
+ if (m_AbsoluteMouseMode) {
+ sendAbsoluteMousePosition(event->x, event->y);
+ }
+
LiSendMouseButtonEvent(event->state == SDL_PRESSED ?
BUTTON_ACTION_PRESS :
BUTTON_ACTION_RELEASE,
@@ -101,81 +107,78 @@
return;
}
- // Batch all pending mouse motion events to save CPU time
- Sint32 x = event->x, y = event->y, xrel = event->xrel, yrel = event->yrel;
- SDL_Event nextEvent;
- while (SDL_PeepEvents(&nextEvent, 1, SDL_GETEVENT, SDL_MOUSEMOTION, SDL_MOUSEMOTION) > 0) {
- event = &nextEvent.motion;
-
- // Ignore synthetic mouse events
- if (event->which != SDL_TOUCH_MOUSEID) {
- x = event->x;
- y = event->y;
- xrel += event->xrel;
- yrel += event->yrel;
- }
- }
+ // Coalesce only adjacent motions. Never consume motion past a click/key
+ // or render event, and bound work so a fast mouse cannot starve rendering.
+ SDL_MouseMotionEvent batched = *event;
+ vdiBatchMouseMotion(batched);
+ Sint32 x = batched.x, y = batched.y;
+ Sint32 xrel = batched.xrel, yrel = batched.yrel;
// We should not reference the original event anymore
event = nullptr;
if (m_AbsoluteMouseMode) {
- int windowWidth, windowHeight;
- SDL_GetWindowSize(m_Window, &windowWidth, &windowHeight);
-
- SDL_Rect src, dst;
- bool mouseInVideoRegion;
-
- src.x = src.y = 0;
- src.w = m_StreamWidth;
- src.h = m_StreamHeight;
-
- dst.x = dst.y = 0;
- dst.w = windowWidth;
- dst.h = windowHeight;
-
- // Use the stream and window sizes to determine the video region
- StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
-
- mouseInVideoRegion = isMouseInVideoRegion(x, y, windowWidth, windowHeight);
-
- // Clamp motion to the video region
- x = qMin(qMax(x - dst.x, 0), dst.w);
- y = qMin(qMax(y - dst.y, 0), dst.h);
-
- // Send the mouse position update if one of the following is true:
- // a) it is in the video region now
- // b) it just left the video region (to ensure the mouse is clamped to the video boundary)
- // c) a mouse button is still down from before the cursor left the video region (to allow smooth dragging)
- Uint32 buttonState = SDL_GetMouseState(nullptr, nullptr);
- if (buttonState == 0) {
- if (m_PendingMouseButtonsAllUpOnVideoRegionLeave) {
- if (m_NeedsManualCaptureOnLeave) {
- // Stop capturing the mouse now
- SDL_CaptureMouse(SDL_FALSE);
- }
- m_PendingMouseButtonsAllUpOnVideoRegionLeave = false;
- }
- }
- if (mouseInVideoRegion || m_MouseWasInVideoRegion || m_PendingMouseButtonsAllUpOnVideoRegionLeave) {
- LiSendMousePositionEvent((short)x, (short)y, dst.w, dst.h);
- }
-
- // Adjust the cursor visibility if applicable
- if (mouseInVideoRegion ^ m_MouseWasInVideoRegion) {
- SDL_ShowCursor((mouseInVideoRegion && m_MouseCursorCapturedVisibilityState == SDL_DISABLE) ? SDL_DISABLE : SDL_ENABLE);
- if (!mouseInVideoRegion && buttonState != 0) {
- // If we still have a button pressed on leave, wait for that to come up
- // before we stop sending mouse position events.
- m_PendingMouseButtonsAllUpOnVideoRegionLeave = true;
- }
- }
-
- m_MouseWasInVideoRegion = mouseInVideoRegion;
+ sendAbsoluteMousePosition(x, y);
}
else {
LiSendMouseMoveEvent(xrel, yrel);
}
+}
+
+void SdlInputHandler::sendAbsoluteMousePosition(Sint32 x, Sint32 y)
+{
+ int windowWidth, windowHeight;
+ SDL_GetWindowSize(m_Window, &windowWidth, &windowHeight);
+
+ SDL_Rect src, dst;
+ bool mouseInVideoRegion;
+
+ src.x = src.y = 0;
+ src.w = m_StreamWidth;
+ src.h = m_StreamHeight;
+
+ dst.x = dst.y = 0;
+ dst.w = windowWidth;
+ dst.h = windowHeight;
+
+ // Use the stream and window sizes to determine the video region
+ StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
+
+ mouseInVideoRegion = isMouseInVideoRegion(x, y, windowWidth, windowHeight);
+
+ // Clamp motion to the video region
+ x = qMin(qMax(x - dst.x, 0), dst.w);
+ y = qMin(qMax(y - dst.y, 0), dst.h);
+
+ // Send the mouse position update if one of the following is true:
+ // a) it is in the video region now
+ // b) it just left the video region (to ensure the mouse is clamped to the video boundary)
+ // c) a mouse button is still down from before the cursor left the video region (to allow smooth dragging)
+ Uint32 buttonState = SDL_GetMouseState(nullptr, nullptr);
+ if (buttonState == 0) {
+ if (m_PendingMouseButtonsAllUpOnVideoRegionLeave) {
+ if (m_NeedsManualCaptureOnLeave) {
+ // Stop capturing the mouse now
+ SDL_CaptureMouse(SDL_FALSE);
+ }
+ m_PendingMouseButtonsAllUpOnVideoRegionLeave = false;
+ }
+ }
+ if (mouseInVideoRegion || m_MouseWasInVideoRegion || m_PendingMouseButtonsAllUpOnVideoRegionLeave) {
+ LiSendMousePositionEvent((short)x, (short)y, dst.w, dst.h);
+ }
+
+ // Adjust the cursor visibility if applicable
+ if (mouseInVideoRegion ^ m_MouseWasInVideoRegion) {
+ SDL_ShowCursor((mouseInVideoRegion && m_MouseCursorCapturedVisibilityState == SDL_DISABLE) ? SDL_DISABLE : SDL_ENABLE);
+ if (!mouseInVideoRegion && buttonState != 0) {
+ // If we still have a button pressed on leave, wait for that to come up
+ // before we stop sending mouse position events.
+ m_PendingMouseButtonsAllUpOnVideoRegionLeave = true;
+ }
+ }
+
+ m_MouseWasInVideoRegion = mouseInVideoRegion;
}
void SdlInputHandler::handleMouseWheelEvent(SDL_MouseWheelEvent* event)
--- a/app/streaming/input/input.h
+++ b/app/streaming/input/input.h
@@ -95,6 +95,8 @@
void handleMouseButtonEvent(SDL_MouseButtonEvent* event);
void handleMouseMotionEvent(SDL_MouseMotionEvent* event);
+
+ void sendAbsoluteMousePosition(Sint32 x, Sint32 y);
void handleMouseWheelEvent(SDL_MouseWheelEvent* event);
--- /dev/null
+++ b/app/streaming/input/vdi_mouse_batch.h
@@ -0,0 +1,22 @@
+#pragma once
+#include <SDL.h>
+
+// Only the SDL main thread consumes events; other threads may append events.
+inline void vdiBatchMouseMotion(SDL_MouseMotionEvent& motion)
+{
+ for (int i = 0; i < 64; ++i) {
+ SDL_Event next;
+ if (SDL_PeepEvents(&next, 1, SDL_PEEKEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) != 1 ||
+ next.type != SDL_MOUSEMOTION || next.motion.which != motion.which ||
+ next.motion.windowID != motion.windowID || next.motion.state != motion.state) {
+ break;
+ }
+ if (SDL_PeepEvents(&next, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) != 1) {
+ break;
+ }
+ motion.x = next.motion.x;
+ motion.y = next.motion.y;
+ motion.xrel += next.motion.xrel;
+ motion.yrel += next.motion.yrel;
+ }
+}
--- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
+++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp
@@ -1,4 +1,5 @@
#include "pacer.h"
+#include "vdi_latest_frame.h"
#include "streaming/streamutils.h"
#ifdef Q_OS_WIN32
@@ -34,6 +35,7 @@
m_VsyncThread(nullptr),
m_DeferredFreeFrame(nullptr),
m_Stopping(false),
+ m_LatestFrameOnly(qEnvironmentVariableIntValue("VDI_LATEST_FRAME") == 1),
m_VsyncSource(nullptr),
m_VsyncRenderer(renderer),
m_MaxVideoFps(0),
@@ -178,7 +180,14 @@
void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame)
{
- dropFrameForEnqueue(m_RenderQueue);
+ if (m_LatestFrameOnly && m_VsyncSource == nullptr) {
+ // Drop only decoded, never presented frames. Keep the decoder reference
+ // chain and the deferred free of the last presented frame intact.
+ m_VideoStats->pacerDroppedFrames += vdiDiscardQueuedFrames(m_RenderQueue);
+ }
+ else {
+ dropFrameForEnqueue(m_RenderQueue);
+ }
m_RenderQueue.enqueue(frame);
m_FrameQueueLock.unlock();
@@ -261,6 +270,8 @@
bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing)
{
+ SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "VDI latest-frame queue: %s",
+ (m_LatestFrameOnly && !enablePacing) ? "enabled" : "disabled");
m_MaxVideoFps = maxVideoFps;
m_DisplayFps = StreamUtils::getDisplayRefreshRate(window);
m_RendererAttributes = m_VsyncRenderer->getRendererAttributes();
--- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h
+++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h
@@ -68,6 +68,7 @@
SDL_Thread* m_VsyncThread;
AVFrame* m_DeferredFreeFrame;
bool m_Stopping;
+ bool m_LatestFrameOnly;
IVsyncSource* m_VsyncSource;
IFFmpegRenderer* m_VsyncRenderer;
--- /dev/null
+++ b/app/streaming/video/ffmpeg-renderers/pacer/vdi_latest_frame.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <QQueue>
+extern "C" {
+#include <libavutil/frame.h>
+}
+
+inline unsigned vdiDiscardQueuedFrames(QQueue<AVFrame*>& queue)
+{
+ unsigned dropped = 0;
+ while (!queue.isEmpty()) {
+ AVFrame* stale = queue.dequeue();
+ av_frame_free(&stale);
+ ++dropped;
+ }
+ return dropped;
+}
@@ -0,0 +1,143 @@
--- a/src/InputStream.c
+++ b/src/InputStream.c
@@ -86,11 +86,16 @@
int deltaX, deltaY;
bool dirty; // Update ready to send (queued packet holder in packetQueue)
} currentRelativeMouseState;
-static struct {
- int x, y;
- int width, height;
- bool dirty; // Update ready to send (queued packet holder in packetQueue)
-} currentAbsoluteMouseState;
+// Only this packet may absorb newer positions. A button event seals the batch.
+static PPACKET_HOLDER currentQueuedAbsoluteMousePacket;
+
+static void latchAbsoluteMousePacket(PPACKET_HOLDER holder) {
+ PltLockMutex(&batchedInputMutex);
+ if (currentQueuedAbsoluteMousePacket == holder) {
+ currentQueuedAbsoluteMousePacket = NULL;
+ }
+ PltUnlockMutex(&batchedInputMutex);
+}
// Initializes the input stream
int initializeInputStream(void) {
@@ -120,7 +125,7 @@
memset(currentGamepadSensorState, 0, sizeof(currentGamepadSensorState));
memset(&currentRelativeMouseState, 0, sizeof(currentRelativeMouseState));
- memset(&currentAbsoluteMouseState, 0, sizeof(currentAbsoluteMouseState));
+ currentQueuedAbsoluteMousePacket = NULL;
PltCreateMutex(&batchedInputMutex);
return 0;
@@ -444,24 +449,9 @@
now = PltGetMillis();
}
- PltLockMutex(&batchedInputMutex);
-
- // Populate the packet with the latest state
- holder->packet.mouseMoveAbs.x = BE16(currentAbsoluteMouseState.x);
- holder->packet.mouseMoveAbs.y = BE16(currentAbsoluteMouseState.y);
-
- // There appears to be a rounding error in GFE's scaling calculation which prevents
- // the cursor from reaching the far edge of the screen when streaming at smaller
- // resolutions with a higher desktop resolution (like streaming 720p with a desktop
- // resolution of 1080p, or streaming 720p/1080p with a desktop resolution of 4K).
- // Subtracting one from the reference dimensions seems to work around this issue.
- holder->packet.mouseMoveAbs.width = BE16(currentAbsoluteMouseState.width - 1);
- holder->packet.mouseMoveAbs.height = BE16(currentAbsoluteMouseState.height - 1);
-
- // The state change is no longer pending
- currentAbsoluteMouseState.dirty = false;
-
- PltUnlockMutex(&batchedInputMutex);
+ // Payload was filled by the producer under the same mutex. Latch
+ // only our packet; a later batch may already be queued after a click.
+ latchAbsoluteMousePacket(holder);
lastMousePacketTime = now;
}
@@ -780,52 +770,37 @@
}
PltLockMutex(&batchedInputMutex);
-
- // Overwrite the previous mouse location with the new one
- currentAbsoluteMouseState.x = x;
- currentAbsoluteMouseState.y = y;
- currentAbsoluteMouseState.width = referenceWidth;
- currentAbsoluteMouseState.height = referenceHeight;
-
- // Queue a packet holder if this is the only pending absolute mouse event
- if (!currentAbsoluteMouseState.dirty) {
- // Set the dirty flag to claim ownership of inserting the packet holder
- // and unlock to allow other threads to enqueue or process input.
- currentAbsoluteMouseState.dirty = true;
- PltUnlockMutex(&batchedInputMutex);
-
+ holder = currentQueuedAbsoluteMousePacket;
+ bool newPacket = holder == NULL;
+ if (newPacket) {
holder = allocatePacketHolder(0);
if (holder == NULL) {
- currentAbsoluteMouseState.dirty = false;
+ PltUnlockMutex(&batchedInputMutex);
return -1;
}
-
holder->channelId = CTRL_CHANNEL_MOUSE;
-
- // TODO: Send this as unreliable sequenced when we have a delayed reliable retransmission thread
holder->enetPacketFlags = ENET_PACKET_FLAG_RELIABLE;
-
holder->packet.mouseMoveAbs.header.size = BE32(sizeof(NV_ABS_MOUSE_MOVE_PACKET) - sizeof(uint32_t));
holder->packet.mouseMoveAbs.header.magic = LE32(MOUSE_MOVE_ABS_MAGIC);
holder->packet.mouseMoveAbs.unused = 0;
-
- // Remaining fields are set in the input thread based on the latest currentAbsoluteMouseState values
-
+ }
+ holder->packet.mouseMoveAbs.x = BE16(x);
+ holder->packet.mouseMoveAbs.y = BE16(y);
+ // Preserve upstream's far-edge rounding workaround.
+ holder->packet.mouseMoveAbs.width = BE16(referenceWidth - 1);
+ holder->packet.mouseMoveAbs.height = BE16(referenceHeight - 1);
+ err = 0;
+ if (newPacket) {
+ currentQueuedAbsoluteMousePacket = holder;
err = LbqOfferQueueItem(&packetQueue, holder, &holder->entry);
if (err != LBQ_SUCCESS) {
+ currentQueuedAbsoluteMousePacket = NULL;
LC_ASSERT(err == LBQ_BOUND_EXCEEDED);
Limelog("Input queue reached maximum size limit\n");
freePacketHolder(holder);
-
- // We weren't able to insert the entry, so let the next call try again
- currentAbsoluteMouseState.dirty = false;
- }
- }
- else {
- // There's already a packet holder queued to send this event
- PltUnlockMutex(&batchedInputMutex);
- err = 0;
- }
+ }
+ }
+ PltUnlockMutex(&batchedInputMutex);
// This is not thread safe, but it's not a big deal because callers that want to
// use LiSendRelativeMotionAsMousePositionEvent() must not mix these function
@@ -872,7 +847,11 @@
holder->packet.mouseButton.header.magic = LE32(holder->packet.mouseButton.header.magic);
holder->packet.mouseButton.button = (uint8_t)button;
+ PltLockMutex(&batchedInputMutex);
+ // Do not let a later position overwrite the move preceding this click.
+ currentQueuedAbsoluteMousePacket = NULL;
err = LbqOfferQueueItem(&packetQueue, holder, &holder->entry);
+ PltUnlockMutex(&batchedInputMutex);
if (err != LBQ_SUCCESS) {
LC_ASSERT(err == LBQ_BOUND_EXCEEDED);
Limelog("Input queue reached maximum size limit\n");
@@ -0,0 +1,179 @@
--- a/app/streaming/session.cpp
+++ b/app/streaming/session.cpp
@@ -39,6 +39,7 @@
#include <QImage>
#include <QGuiApplication>
#include <QCursor>
+#include "input/vdi_input_priority.h"
#include <QScreen>
#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
@@ -1962,8 +1963,14 @@
// Hijack this thread to be the SDL main thread. We have to do this
// because we want to suspend all Qt processing until the stream is over.
+ VdiInputPriority vdiInputPriority(qEnvironmentVariableIntValue("VDI_INPUT_PRIORITY") == 1,
+ SDL_CODE_FRAME_READY);
+ VdiInputTiming vdiInputTiming(qEnvironmentVariableIntValue("VDI_INPUT_TIMING") == 1);
+ SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "VDI input priority: %s",
+ qEnvironmentVariableIntValue("VDI_INPUT_PRIORITY") == 1 ? "enabled" : "disabled");
SDL_Event event;
for (;;) {
+ if (!vdiInputPriority.takeDeferred(event, SDL_GetWindowID(m_Window))) {
#if SDL_VERSION_ATLEAST(2, 0, 18) && !defined(STEAM_LINK)
// SDL 2.0.18 has a proper wait event implementation that uses platform
// support to block on events rather than polling on Windows, macOS, X11,
@@ -1995,6 +2002,11 @@
continue;
}
#endif
+ if (vdiInputPriority.deferFrame(event)) {
+ continue;
+ }
+ }
+ vdiInputTiming.observe(event, SDL_GetWindowID(m_Window));
switch (event.type) {
case SDL_QUIT:
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
@@ -2005,7 +2017,9 @@
switch (event.user.code) {
case SDL_CODE_FRAME_READY:
if (m_VideoDecoder != nullptr) {
+ const auto renderStart = vdiInputTiming.startRender();
m_VideoDecoder->renderFrameOnMainThread();
+ vdiInputTiming.endRender(renderStart);
}
break;
case SDL_CODE_FLUSH_WINDOW_EVENT_BARRIER:
@@ -2308,6 +2322,7 @@
}
DispatchDeferredCleanup:
+ vdiInputTiming.report(vdiInputPriority.prioritized());
// Switch back to synchronous logging mode
StreamUtils::exitAsyncLoggingMode();
--- /dev/null
+++ b/app/streaming/input/vdi_input_priority.h
@@ -0,0 +1,121 @@
+#pragma once
+#include <SDL.h>
+#include <array>
+#include <algorithm>
+#include <cstdint>
+
+// SDL main thread only. Inputs may pass one frame-ready notification, never a
+// window/focus/quit/other notification or another frame. Input FIFO is preserved.
+class VdiInputPriority {
+public:
+ VdiInputPriority(bool enabled, Sint32 frameCode)
+ : m_Enabled(enabled), m_FrameCode(frameCode) {}
+
+ bool deferFrame(const SDL_Event& event) {
+ if (!m_Enabled || event.type != SDL_USEREVENT || event.user.code != m_FrameCode)
+ return false;
+ SDL_assert(!m_Pending);
+ m_Frame = event;
+ m_Pending = true;
+ m_Count = 0;
+ m_Start = SDL_GetPerformanceCounter();
+ return true;
+ }
+
+ bool takeDeferred(SDL_Event& event, Uint32 windowId) {
+ if (!m_Pending) return false;
+ SDL_Event next;
+ const auto elapsed = SDL_GetPerformanceCounter() - m_Start;
+ if (m_Count < 32 && elapsed < SDL_GetPerformanceFrequency() / 500 &&
+ SDL_PeepEvents(&next, 1, SDL_PEEKEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) == 1 &&
+ isInputForWindow(next, windowId) &&
+ SDL_PeepEvents(&event, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) == 1) {
+ ++m_Count;
+ ++m_Prioritized;
+ return true;
+ }
+ event = m_Frame;
+ m_Pending = false;
+ return true;
+ }
+
+ uint64_t prioritized() const { return m_Prioritized; }
+
+ static bool isInputForWindow(const SDL_Event& event, Uint32 windowId) {
+ switch (event.type) {
+ case SDL_MOUSEMOTION:
+ return event.motion.windowID == windowId && event.motion.which != SDL_TOUCH_MOUSEID;
+ case SDL_MOUSEBUTTONDOWN:
+ case SDL_MOUSEBUTTONUP:
+ return event.button.windowID == windowId && event.button.which != SDL_TOUCH_MOUSEID;
+ case SDL_MOUSEWHEEL:
+ return event.wheel.windowID == windowId && event.wheel.which != SDL_TOUCH_MOUSEID;
+ case SDL_KEYDOWN:
+ case SDL_KEYUP:
+ return event.key.windowID == windowId;
+ default:
+ return false;
+ }
+ }
+
+private:
+ bool m_Enabled;
+ Sint32 m_FrameCode;
+ bool m_Pending = false;
+ SDL_Event m_Frame{};
+ unsigned m_Count = 0;
+ Uint64 m_Start = 0;
+ uint64_t m_Prioritized = 0;
+};
+
+// Aggregated only when explicitly requested. Ages are SDL queue/dispatch ages,
+// not network RTT or physical input-to-photon latency. 0.1 ms histogram bins.
+class VdiInputTiming {
+public:
+ explicit VdiInputTiming(bool enabled) : m_Enabled(enabled) {}
+ void observe(const SDL_Event& event, Uint32 windowId) {
+ if (!m_Enabled || !event.common.timestamp ||
+ !VdiInputPriority::isInputForWindow(event, windowId)) return;
+ const Uint32 age = SDL_GetTicks() - event.common.timestamp;
+ // Ignore invalid/future synthetic timestamps; subtraction handles wrap.
+ if (age < 60000) m_Input.add(Uint64(age) * 10);
+ }
+ Uint64 startRender() const { return m_Enabled ? SDL_GetPerformanceCounter() : 0; }
+ void endRender(Uint64 start) {
+ if (m_Enabled) m_Render.add((SDL_GetPerformanceCounter() - start) * 10000 / SDL_GetPerformanceFrequency());
+ }
+ void report(uint64_t prioritized) const {
+ if (!m_Enabled) return;
+ SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+ "VDI input timing: SDL dispatch age only; prioritized=%llu",
+ (unsigned long long)prioritized);
+ m_Input.report("input dispatch age");
+ m_Render.report("main-thread render call");
+ }
+private:
+ struct Histogram {
+ std::array<uint64_t, 1024> bins{};
+ uint64_t count = 0, total = 0, max = 0;
+ void add(uint64_t tenths) {
+ ++bins[std::min<uint64_t>(tenths, bins.size()-1)];
+ ++count; total += tenths; max = std::max(max, tenths);
+ }
+ double quantile(unsigned percent) const {
+ const uint64_t rank = (count * percent + 99) / 100;
+ uint64_t accumulated = 0;
+ for (size_t i = 0; i < bins.size(); ++i) {
+ accumulated += bins[i];
+ if (accumulated >= rank) return i / 10.0;
+ }
+ return 0;
+ }
+ void report(const char* label) const {
+ SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+ "VDI %s: samples=%llu avg=%.2f p50=%.1f p95=%.1f max=%.1f ms (percentiles capped at 102.3ms)",
+ label, (unsigned long long)count, count ? total / (10.0*count) : 0,
+ count ? quantile(50) : 0, count ? quantile(95) : 0, max / 10.0);
+ }
+ };
+ bool m_Enabled;
+ Histogram m_Input, m_Render;
+};
@@ -0,0 +1,63 @@
// Include the production implementation so batching can be inspected without
// starting a real network connection. Unused transport code is linker-collected.
#include "InputStream.c"
#undef NDEBUG
#include <assert.h>
static PPACKET_HOLDER pop(void) {
PPACKET_HOLDER holder = NULL;
assert(LbqPollQueueElement(&packetQueue, (void**)&holder) == LBQ_SUCCESS);
return holder;
}
static void expectPosition(int x) {
PPACKET_HOLDER holder = pop();
assert(holder->packet.header.magic == LE32(MOUSE_MOVE_ABS_MAGIC));
latchAbsoluteMousePacket(holder);
assert(holder->packet.mouseMoveAbs.x == BE16(x));
assert((uint16_t)holder->packet.mouseMoveAbs.width == (uint16_t)BE16(2559));
freePacketHolder(holder);
}
int main(void) {
AppVersionQuad[0] = 7;
LbqInitializeLinkedBlockingQueue(&packetQueue, MAX_QUEUED_INPUT_PACKETS);
LbqInitializeLinkedBlockingQueue(&packetHolderFreeList, MAX_QUEUED_INPUT_PACKETS);
PltCreateMutex(&batchedInputMutex);
initialized = true;
for (int i = 0; i < 1000; ++i) {
// A deliberately stalled sender must preserve each click's coordinates.
assert(LiSendMousePositionEvent(10, 10, 2560, 1440) == 0);
assert(LiSendMousePositionEvent(20, 20, 2560, 1440) == 0);
assert(LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT) == 0);
assert(LiSendMousePositionEvent(30, 30, 2560, 1440) == 0);
assert(LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT) == 0);
assert(LiSendMousePositionEvent(40, 40, 2560, 1440) == 0);
expectPosition(20);
PPACKET_HOLDER press = pop();
assert(press->packet.header.magic == LE32(BUTTON_ACTION_PRESS + 1));
freePacketHolder(press);
expectPosition(30);
PPACKET_HOLDER release = pop();
assert(release->packet.header.magic == LE32(BUTTON_ACTION_RELEASE + 1));
freePacketHolder(release);
// Latching an older packet above must not prevent the latest batch
// from still absorbing newer positions.
assert(LiSendMousePositionEvent(50, 50, 2560, 1440) == 0);
expectPosition(50);
PPACKET_HOLDER empty;
assert(LbqPollQueueElement(&packetQueue, (void**)&empty) == LBQ_NO_ELEMENT);
}
initialized = false;
PLINKED_BLOCKING_QUEUE_ENTRY entry = LbqDestroyLinkedBlockingQueue(&packetQueue);
assert(entry == NULL);
entry = LbqDestroyLinkedBlockingQueue(&packetHolderFreeList);
while (entry) {
PLINKED_BLOCKING_QUEUE_ENTRY next = entry->flink;
free(entry->data);
entry = next;
}
PltDeleteMutex(&batchedInputMutex);
puts("PASS: 1000 queued move/press/move/release cycles preserve positions");
return 0;
}
@@ -0,0 +1,86 @@
#include "vdi_input_priority.h"
#undef NDEBUG
#include <cassert>
#include <cstdio>
constexpr Sint32 frameCode = 123;
static SDL_Event frame() {
SDL_Event e{}; e.type = SDL_USEREVENT; e.user.code = frameCode; return e;
}
static SDL_Event input(Uint32 type, Uint32 window = 1) {
SDL_Event e{}; e.type = type; e.key.windowID = window; return e;
}
static void push(SDL_Event e) { assert(SDL_PushEvent(&e) == 1); }
static SDL_Event pop() {
SDL_Event e{};
assert(SDL_PeepEvents(&e,1,SDL_GETEVENT,SDL_FIRSTEVENT,SDL_LASTEVENT)==1);
return e;
}
int main() {
assert(SDL_Init(SDL_INIT_EVENTS)==0);
SDL_Event out{};
VdiInputPriority disabled(false, frameCode);
push(input(SDL_MOUSEMOTION));
assert(!disabled.deferFrame(frame()) && !disabled.takeDeferred(out,1));
assert(pop().type == SDL_MOUSEMOTION);
// Only a contiguous input prefix can pass a frame. No focus/quit/decoder
// barrier may be skipped, even if input is waiting further down the queue.
for (Uint32 barrier : {Uint32(SDL_WINDOWEVENT),Uint32(SDL_QUIT),
Uint32(SDL_USEREVENT),Uint32(SDL_RENDER_DEVICE_RESET),
Uint32(SDL_CONTROLLERDEVICEADDED)}) {
VdiInputPriority scheduler(true, frameCode);
push(input(SDL_MOUSEBUTTONDOWN)); push(input(SDL_MOUSEBUTTONUP));
push(input(barrier)); push(input(SDL_KEYDOWN));
assert(scheduler.deferFrame(frame()));
assert(scheduler.takeDeferred(out,1) && out.type==SDL_MOUSEBUTTONDOWN);
assert(scheduler.takeDeferred(out,1) && out.type==SDL_MOUSEBUTTONUP);
assert(scheduler.takeDeferred(out,1) && out.type==SDL_USEREVENT && out.user.code==frameCode);
assert(!scheduler.takeDeferred(out,1));
assert(pop().type==barrier && pop().type==SDL_KEYDOWN);
assert(scheduler.prioritized()==2);
}
// Preserve all input order, including keyboard releases and drag edges.
{
VdiInputPriority scheduler(true,frameCode);
for (Uint32 t : {Uint32(SDL_KEYDOWN),Uint32(SDL_MOUSEMOTION),Uint32(SDL_MOUSEBUTTONDOWN),
Uint32(SDL_MOUSEMOTION),Uint32(SDL_MOUSEBUTTONUP),Uint32(SDL_KEYUP)}) push(input(t));
assert(scheduler.deferFrame(frame()));
for (Uint32 t : {Uint32(SDL_KEYDOWN),Uint32(SDL_MOUSEMOTION),Uint32(SDL_MOUSEBUTTONDOWN),
Uint32(SDL_MOUSEMOTION),Uint32(SDL_MOUSEBUTTONUP),Uint32(SDL_KEYUP)})
assert(scheduler.takeDeferred(out,1) && out.type==t);
assert(scheduler.takeDeferred(out,1) && out.type==SDL_USEREVENT);
}
for (bool synthetic : {false,true}) {
VdiInputPriority scheduler(true,frameCode);
auto e=input(SDL_MOUSEMOTION,synthetic?1:2);
if(synthetic) e.motion.which=SDL_TOUCH_MOUSEID;
push(e); assert(scheduler.deferFrame(frame()));
assert(scheduler.takeDeferred(out,1) && out.type==SDL_USEREVENT);
assert(pop().type==SDL_MOUSEMOTION);
}
// A high-rate input device cannot starve video.
{
VdiInputPriority scheduler(true,frameCode);
for(int i=0;i<1000;++i) push(input(SDL_MOUSEWHEEL));
assert(scheduler.deferFrame(frame()));
unsigned n=0;
while(scheduler.takeDeferred(out,1) && out.type!=SDL_USEREVENT) ++n;
assert(n<=32 && n>0 && out.type==SDL_USEREVENT);
SDL_FlushEvents(SDL_FIRSTEVENT,SDL_LASTEVENT);
}
{
VdiInputPriority scheduler(true,frameCode);
push(input(SDL_MOUSEBUTTONUP));
assert(scheduler.deferFrame(frame())); SDL_Delay(4);
assert(scheduler.takeDeferred(out,1) && out.type==SDL_USEREVENT);
assert(pop().type==SDL_MOUSEBUTTONUP);
}
// Instrumentation never consumes events or changes cursor visibility.
VdiInputTiming timing(true);
auto e=input(SDL_MOUSEMOTION); e.common.timestamp=SDL_GetTicks();
timing.observe(e,1); auto start=timing.startRender(); timing.endRender(start);
timing.report(0);
SDL_Quit();
std::puts("PASS: input FIFO, focus/quit/reset/window barriers, disabled mode, device/window isolation, count/time bounds");
}
+75
View File
@@ -0,0 +1,75 @@
#include <SDL.h>
#include <cstdio>
#include "vdi_mouse_batch.h"
#include "vdi_latest_frame.h"
#undef NDEBUG
#include <cassert>
static SDL_Event motion(int x, Uint32 which = 1)
{
SDL_Event event{};
event.type = SDL_MOUSEMOTION;
event.motion.which = which;
event.motion.windowID = 1;
event.motion.x = x;
event.motion.y = 100;
event.motion.xrel = 1;
return event;
}
int main()
{
assert(SDL_Init(SDL_INIT_EVENTS) == 0);
for (Uint32 boundary : {Uint32(SDL_MOUSEBUTTONDOWN), Uint32(SDL_MOUSEBUTTONUP),
Uint32(SDL_KEYDOWN), Uint32(SDL_USEREVENT)}) {
SDL_FlushEvents(SDL_FIRSTEVENT, SDL_LASTEVENT);
SDL_Event first = motion(10), adjacent = motion(20), later = motion(90), barrier{};
barrier.type = boundary;
assert(SDL_PushEvent(&adjacent) == 1);
assert(SDL_PushEvent(&barrier) == 1);
assert(SDL_PushEvent(&later) == 1);
vdiBatchMouseMotion(first.motion);
assert(first.motion.x == 20 && first.motion.xrel == 2);
SDL_Event next;
assert(SDL_PeepEvents(&next, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) == 1);
assert(next.type == boundary);
assert(SDL_PeepEvents(&next, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) == 1);
assert(next.motion.x == 90);
}
SDL_FlushEvents(SDL_FIRSTEVENT, SDL_LASTEVENT);
SDL_Event first = motion(0);
for (int i = 1; i <= 200; ++i) {
SDL_Event next = motion(i);
assert(SDL_PushEvent(&next) == 1);
}
vdiBatchMouseMotion(first.motion);
assert(first.motion.x == 64); // Work per dispatch is bounded.
assert(SDL_PeepEvents(nullptr, 0, SDL_PEEKEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT) == 136);
SDL_FlushEvents(SDL_FIRSTEVENT, SDL_LASTEVENT);
first = motion(10);
SDL_Event synthetic = motion(99, SDL_TOUCH_MOUSEID);
assert(SDL_PushEvent(&synthetic) == 1);
vdiBatchMouseMotion(first.motion);
assert(first.motion.x == 10);
AVFrame* presented = av_frame_alloc();
assert(presented);
presented->buf[0] = av_buffer_alloc(16);
assert(presented->buf[0]);
QQueue<AVFrame*> queue;
assert(vdiDiscardQueuedFrames(queue) == 0);
for (int i = 0; i < 200; ++i) {
unsigned dropped = vdiDiscardQueuedFrames(queue);
assert(dropped == (i == 0 ? 0u : 1u));
AVFrame* frame = av_frame_alloc();
assert(frame);
frame->pts = i;
queue.enqueue(frame);
assert(queue.size() == 1 && queue.head()->pts == i);
assert(av_buffer_get_ref_count(presented->buf[0]) == 1);
}
assert(vdiDiscardQueuedFrames(queue) == 1);
av_frame_free(&presented);
SDL_Quit();
std::puts("PASS: SDL event boundaries, bounded motion work, latest decoded frame ownership");
}