diff --git a/Assets/Editor/BodyTrackingSetup.cs b/Assets/Editor/BodyTrackingSetup.cs new file mode 100644 index 0000000..40476e6 --- /dev/null +++ b/Assets/Editor/BodyTrackingSetup.cs @@ -0,0 +1,520 @@ +using UnityEditor; +using UnityEngine; + +/// +/// Enables Meta IOBT body tracking in the project settings. +/// +/// These live in serialized assets that are inconvenient to edit by hand, and getting any one of +/// them wrong fails in a misleading way: the app runs, every inspector checkbox looks right, and +/// OVRBody still reports "Failed to start body tracking". Scripted so it is reproducible. +/// +/// Unity -batchmode -nographics -projectPath <proj> -buildTarget Android \ +/// -executeMethod BodyTrackingSetup.Configure -quit +/// +/// or from the editor: Tools > Body Tracking > Configure Project Settings. +/// +public static class BodyTrackingSetup +{ + [MenuItem("Tools/Body Tracking/Configure Project Settings")] + public static void Configure() + { + ConfigureProjectConfig(); + ConfigureRuntimeSettings(); + // Runs last: it opens and saves scenes, so anything editing scene objects must come before + // ConfigureSceneBodySource saves, and ConfigurePermissionRequest already opens scenes too. + ConfigurePermissionRequest(); + ConfigureSceneBodySource(); + AssetDatabase.SaveAssets(); + Debug.Log("[BodyTrackingSetup] done"); + } + + + + + /// + /// Single debug APK at Build/quest-body.apk, for iterating on body tracking. + /// + /// Deliberately separate from ProjectBuild.Build(): that one produces four APKs + /// (cn/i18n x debug/release) and swaps region jars, which is release packaging rather than + /// something to run on every code change. Keep BuildOptions.Development so Debug.Log + /// survives into logcat. + /// + [MenuItem("Tools/Body Tracking/Build Debug APK")] + public static void BuildDebugApk() + { + var scenes = new System.Collections.Generic.List(); + foreach (var s in EditorBuildSettings.scenes) + { + if (s != null && s.enabled) + { + scenes.Add(s.path); + } + } + + if (scenes.Count == 0) + { + Debug.LogError("[BodyTrackingSetup] no enabled scenes in build settings"); + FailBuild(); + return; + } + + System.IO.Directory.CreateDirectory("Build"); + const string outPath = "Build/quest-body.apk"; + + // Uses the project's own application id. A local build is signed with the debug key, so + // Android will not install it over a release-signed package of the same id -- uninstall + // the official build first: + // adb uninstall com.xrobotoolkit.client.quest + var summary = BuildPipeline.BuildPlayer(new BuildPlayerOptions + { + scenes = scenes.ToArray(), + locationPathName = outPath, + target = BuildTarget.Android, + targetGroup = BuildTargetGroup.Android, + options = BuildOptions.Development, + }).summary; + + Debug.Log($"[BodyTrackingSetup] build {summary.result}: " + + $"{summary.totalSize / 1024 / 1024} MB, {summary.totalErrors} errors, " + + $"took {summary.totalTime}"); + + if (summary.result != UnityEditor.Build.Reporting.BuildResult.Succeeded) + { + FailBuild(); + } + } + + /// + /// Signals build failure without taking an interactive editor down with it. + /// + /// + /// A non-zero exit is what a CI runner needs, but calling it from the menu would close the + /// editor and lose unsaved work, so it is limited to batch mode. Interactively the logged + /// error is the signal. + /// + private static void FailBuild() + { + if (Application.isBatchMode) + { + EditorApplication.Exit(1); + } + } + + /// Quest Features > Body Tracking Support = Supported. + private static void ConfigureProjectConfig() + { + var cfg = OVRProjectConfig.CachedProjectConfig; + if (cfg == null) + { + Debug.LogError("[BodyTrackingSetup] OVRProjectConfig.CachedProjectConfig == null"); + return; + } + + cfg.bodyTrackingSupport = OVRProjectConfig.FeatureSupport.Supported; + OVRProjectConfig.CommitProjectConfig(cfg); + Debug.Log("[BodyTrackingSetup] bodyTrackingSupport=Supported"); + } + + /// + /// Fidelity=High is what actually turns IOBT on; Low is IK inference from headset and + /// controllers only, not camera-measured limbs. + /// + /// JointSet is only the startup default; the Mode dropdown switches it at runtime. + /// + private static void ConfigureRuntimeSettings() + { + var rs = OVRRuntimeSettings.GetRuntimeSettings(); + rs.BodyTrackingFidelity = OVRPlugin.BodyTrackingFidelity2.High; + // The joint set the app starts on, not a fixed choice: the Mode dropdown switches it at + // runtime via OVRBody.SetRequestedJointSet. UpperBody is the default because its joints are + // all camera-measured, whereas FullBody's extra 14 are inferred by Generative Legs. + rs.BodyTrackingJointSet = OVRPlugin.BodyJointSet.UpperBody; + OVRRuntimeSettings.CommitRuntimeSettings(rs); + Debug.Log($"[BodyTrackingSetup] fidelity={rs.BodyTrackingFidelity} " + + $"jointSet={rs.BodyTrackingJointSet}"); + } + + /// + /// OVRManager > Permission Requests On Startup > Body Tracking. + /// + /// Without this the app never asks for com.oculus.permission.BODY_TRACKING at runtime, so + /// tracking is refused even though the manifest declares the permission. + /// + private static void ConfigurePermissionRequest() + { + // Scoped to Assets/: an unscoped search also returns scenes shipped inside read-only + // packages (e.g. Meta's OVRTransitionScene), and opening one of those throws. + foreach (var guid in AssetDatabase.FindAssets("t:Scene", new[] { "Assets" })) + { + var path = AssetDatabase.GUIDToAssetPath(guid); + var scene = UnityEditor.SceneManagement.EditorSceneManager.OpenScene( + path, UnityEditor.SceneManagement.OpenSceneMode.Single); + if (!scene.IsValid()) + { + continue; + } + + var changed = false; + foreach (var mgr in Object.FindObjectsOfType()) + { + // requestBodyTrackingPermissionOnStartup is internal, so reach it via + // SerializedObject rather than adding a dependency on Meta's internals. + var so = new SerializedObject(mgr); + var prop = so.FindProperty("requestBodyTrackingPermissionOnStartup"); + if (prop == null) + { + Debug.LogWarning($"[BodyTrackingSetup] {mgr.name}: property not found"); + continue; + } + + if (!prop.boolValue) + { + prop.boolValue = true; + so.ApplyModifiedPropertiesWithoutUndo(); + changed = true; + Debug.Log($"[BodyTrackingSetup] {path}: {mgr.name} permission request enabled"); + } + } + + if (changed) + { + UnityEditor.SceneManagement.EditorSceneManager.MarkSceneDirty(scene); + UnityEditor.SceneManagement.EditorSceneManager.SaveScene(scene); + } + } + } + + /// + /// Adds the OVRBody component and wires QuestTrackingDataSource.body / .trackingSpace. + /// + /// Without an OVRBody in the scene nothing ever requests a tracking session from the runtime, + /// so BodyState stays null and no BodyMeta is emitted. This failed silently: the build + /// succeeded, project settings were all correct, and logcat showed no body-tracking errors at + /// all -- just no output. The component goes on the OVRCameraRig so it shares the rig's + /// lifetime, and OVRBody's own _providedSkeletonType default (UpperBody) already matches + /// ConfigureRuntimeSettings. + /// + private static void ConfigureSceneBodySource() + { + foreach (var guid in AssetDatabase.FindAssets("t:Scene", new[] { "Assets" })) + { + var path = AssetDatabase.GUIDToAssetPath(guid); + var scene = UnityEditor.SceneManagement.EditorSceneManager.OpenScene( + path, UnityEditor.SceneManagement.OpenSceneMode.Single); + if (!scene.IsValid()) + { + continue; + } + + var sources = Object.FindObjectsOfType(); + if (sources.Length == 0) + { + continue; + } + + var rig = Object.FindObjectOfType(); + if (rig == null) + { + Debug.LogWarning($"[BodyTrackingSetup] {path}: no OVRCameraRig, skipping"); + continue; + } + + var body = Object.FindObjectOfType(); + if (body == null) + { + body = rig.gameObject.AddComponent(); + Debug.Log($"[BodyTrackingSetup] {path}: added OVRBody to {rig.name}"); + } + + foreach (var src in sources) + { + src.body = body; + src.trackingSpace = rig.trackingSpace; + EditorUtility.SetDirty(src); + Debug.Log($"[BodyTrackingSetup] {path}: {src.name}.body={body.name} " + + $"trackingSpace={rig.trackingSpace?.name ?? "null"}"); + } + + ActivateBodyModeUi(path); + + UnityEditor.SceneManagement.EditorSceneManager.MarkSceneDirty(scene); + UnityEditor.SceneManagement.EditorSceneManager.SaveScene(scene); + } + } + + /// + /// Un-hides the body mode dropdown and every inactive ancestor of it. + /// + /// + /// Upstream ships the "Mode" row deactivated because body tracking was unimplemented on Quest + /// ("Coming soon"). Wiring the dropdown's listeners is not enough on its own: an inactive + /// ancestor keeps the control off-screen, so on-device there is simply no dropdown to select + /// and TrackingType stays None -- which looks identical to a data pipeline that is broken. + /// + /// Walks up from the dropdown rather than hardcoding "Mode", since only the ancestor chain is + /// guaranteed to be what actually gates visibility. + /// + private static void ActivateBodyModeUi(string path) + { + foreach (var ui in Object.FindObjectsOfType(true)) + { + var drop = ui.bodyModeDrop; + if (drop == null) + { + Debug.LogWarning($"[BodyTrackingSetup] {path}: {ui.name}.bodyModeDrop unassigned"); + continue; + } + + for (var t = drop.transform; t != null; t = t.parent) + { + if (t.gameObject.activeSelf) + { + continue; + } + + t.gameObject.SetActive(true); + EditorUtility.SetDirty(t.gameObject); + Debug.Log($"[BodyTrackingSetup] {path}: activated '{t.name}'"); + } + + RelabelBodyModeOptions(drop, path); + WidenBodyModeCaption(drop, path); + LayOutBodyModeRow(drop, path); + WidenBodyInfo(ui, path); + } + } + + /// + /// Gives the Status line room for the full, unabbreviated readout. + /// + /// + /// It was authored at 280px for the old static text ("BodyTracking on (High)"). The live + /// readout spells out joint count, calibration state and confidence, which does not fit; left + /// alone the tail is clipped mid-word, and calibration state is the part that gets cut. + /// + /// Overflow is enabled as well as widening: the widest string is the "enable it in system + /// settings" instruction, which no width available in this panel fits, and spilling that rare + /// case is better than shrinking the text shown constantly. + /// + private static void WidenBodyInfo(UIOperate ui, string path) + { + var info = ui.BodyInfo; + if (info == null) + { + Debug.LogWarning($"[BodyTrackingSetup] {path}: {ui.name}.BodyInfo unassigned"); + return; + } + + var rt = (RectTransform)info.transform; + + // Pivot and anchor on the left edge before widening. Its parent is a zero-size layout node, + // so with the authored centre pivot the text grows symmetrically about a point -- widening + // it pushed half the extra width off the left side of the panel rather than filling the + // empty space to the right. Anchoring left makes the width grow rightwards only. + rt.anchorMin = new Vector2(0f, 0.5f); + rt.anchorMax = new Vector2(0f, 0.5f); + rt.pivot = new Vector2(0f, 0.5f); + // Spans from under the "S" of the Status heading to the panel's right edge. + rt.sizeDelta = new Vector2(265f, rt.sizeDelta.y); + // Lines the readout up with the "S" of the "Status" heading above it, which is how the + // other sections in this panel read: "Mode" sits under the "B" of "Body Tracking", and + // "Send" under the "D" of "Data & Control". The offset is negative because the parent is + // placed by its own layout group well right of the panel edge, so this walks back left + // from there rather than indenting in from the left. + rt.anchoredPosition = new Vector2(-139f, rt.anchoredPosition.y); + info.horizontalOverflow = HorizontalWrapMode.Overflow; + info.alignment = TextAnchor.MiddleLeft; + // Smaller than the panel's labels: it is a readout rather than a control, and spelling the + // fields out in full does not fit between the dropdown's left edge and the panel's right + // edge at the authored 14pt. Shrinking the glyphs keeps the words whole, which is the point. + info.fontSize = 11; + EditorUtility.SetDirty(info); + EditorUtility.SetDirty(rt); + Debug.Log($"[BodyTrackingSetup] {path}: BodyInfo laid out at " + + $"x={rt.anchoredPosition.x} width={rt.sizeDelta.x}"); + } + + /// + /// Places the mode row inside the panel, and drops the "Coming soon ..." placeholder. + /// + /// + /// The row was authored to sit hidden behind that placeholder, so its layout was never + /// meaningful: the resolved rect put the "Mode" label at x=-1.15 against a panel whose left + /// edge is x=-0.86, i.e. the whole control rendered outside the panel, to the left of the + /// window. Two nested HorizontalLayoutGroups were fighting -- the "Mode" group padded +67 + /// while its "BodyTracking" child padded -267, netting -200. + /// + /// Rather than tune those against each other, both are zeroed and the row is aligned on the + /// Head/Controller row above it, which is the visual reference the operator actually compares + /// against. Spacing was likewise a large negative number (-623) cancelling the padding. + /// + /// Kept in the setup script rather than hand-edited into Main.unity so it stays reproducible, + /// and re-running it after an upstream merge repairs the row instead of silently regressing. + /// + private static void LayOutBodyModeRow(UnityEngine.UI.Dropdown drop, string path) + { + // dropdown -> BodyTracking (inner group) -> Mode (outer group) + var inner = drop.transform.parent; + var outer = inner != null ? inner.parent : null; + if (inner == null || outer == null) + { + Debug.LogWarning($"[BodyTrackingSetup] {path}: unexpected mode row hierarchy"); + return; + } + + // Matches the Head/Controller row's group, so the "Mode" label lines up with "Head". + var outerGroup = outer.GetComponent(); + if (outerGroup != null) + { + outerGroup.padding = new RectOffset(1, 0, 0, 0); + outerGroup.spacing = 0; + outerGroup.childForceExpandWidth = false; + EditorUtility.SetDirty(outerGroup); + } + + var innerGroup = inner.GetComponent(); + if (innerGroup != null) + { + // Bottom padding lifts the row; the authored 9px of *top* padding pushed it down + // instead. Measured against the rows this should match: Mode sat 0.034 world units + // below its box centre where Head/Controller/Send sit 0.007-0.010 below theirs. + // Zeroing top got it to 0.021, and 8px of bottom padding covers the rest. + innerGroup.padding = new RectOffset(0, 0, 0, 8); + innerGroup.spacing = 8; + innerGroup.childForceExpandWidth = false; + innerGroup.childAlignment = TextAnchor.MiddleLeft; + EditorUtility.SetDirty(innerGroup); + } + + // The label carries a stale offset from when it was positioned by hand; the layout group + // drives x now, but a non-zero anchoredPosition still shifts it. + foreach (RectTransform child in inner) + { + child.anchoredPosition = new Vector2(0, child.anchoredPosition.y); + EditorUtility.SetDirty(child); + } + + HideComingSoonPlaceholder(outer.parent, path); + Debug.Log($"[BodyTrackingSetup] {path}: mode row re-laid out"); + } + + /// + /// Hides the "Coming soon ..." label that upstream showed in place of the body mode control. + /// + /// + /// Matched on its text rather than its name ("Label (1)") because the name carries no meaning + /// and would silently hide the wrong object if the panel is ever rearranged. Deactivated + /// rather than deleted so the change stays reversible and merges cleanly. + /// + private static void HideComingSoonPlaceholder(Transform panel, string path) + { + if (panel == null) + { + return; + } + + foreach (var text in panel.GetComponentsInChildren(true)) + { + if (text.text == null || !text.text.StartsWith("Coming soon")) + { + continue; + } + + text.gameObject.SetActive(false); + EditorUtility.SetDirty(text.gameObject); + Debug.Log($"[BodyTrackingSetup] {path}: hid placeholder '{text.text}'"); + } + } + + /// + /// Relabels the body mode dropdown to match what the options actually do on Quest. + /// + /// + /// OnBodyModeDrop casts the dropdown *index* to TrackingType, so the labels are decorative + /// while the index is load-bearing. Upstream inherited PICO's "None / Upper / Full" wording, + /// where index 2 ("Full") was PICO's external-tracker mode -- no Quest equivalent, so "Full" + /// read like full-body capture but was a dead end. It now really is full body: index 2 is + /// TrackingType.FullBody, which switches the runtime to the 84-joint skeleton. + /// + /// The joint count is spelled out because it is the one thing that distinguishes the two on + /// the wire, and it is what the status line reports back once a switch takes effect. + /// + /// + /// Gives the dropdown and its labels room for the mode names. + /// + /// + /// The control was authored 70px wide with 45px Wrap-overflow labels, which fitted the old + /// "Body (IOBT)" but clips "Upper Body (70)" -- and a label that reads as a different mode is + /// worse than one that is visibly cut off. + /// + /// The control itself has to grow, not just the text inside it: stretching the labels to the + /// parent only buys 70px, still a few short. Its layout group has ChildControlWidth off and + /// 461px of row to spend, so widening the rect sticks. + /// + /// Widened rather than shrinking the font: at size 10 these are already the smallest text in + /// the panel. + /// + private static void WidenBodyModeCaption(UnityEngine.UI.Dropdown drop, string path) + { + // Fits "Upper Body (70)" at font size 10 with margin for the arrow, measured from the + // clipped result rather than guessed: 70px cut the closing bracket. + const float dropdownWidth = 130f; + var dropRt = (RectTransform)drop.transform; + dropRt.sizeDelta = new Vector2(dropdownWidth, dropRt.sizeDelta.y); + EditorUtility.SetDirty(dropRt); + + // Left inset clearing the item template's checkmark, which is anchored left, 20px wide at + // x=10 -- so it occupies 0..20px. Taking the label to x=0 to gain width put the text + // underneath it and the tick overlapped the first letter. The caption has no checkmark and + // keeps the authored inset, so the two are indented alike. + const float checkmarkInset = 22f; + + // The caption shows the current selection; the item template is what the open list uses. + // Fixing only the caption leaves the list itself clipped. + foreach (var label in new[] { drop.captionText, drop.itemText }) + { + if (label == null) + { + continue; + } + + var isItem = label == drop.itemText; + var inset = isItem ? checkmarkInset : 10f; + var rt = (RectTransform)label.transform; + // Stretch to the parent's width instead of a fixed size, so this does not need to know + // how wide the dropdown is -- and stays right if the row is ever re-laid out. + rt.anchorMin = new Vector2(0f, rt.anchorMin.y); + rt.anchorMax = new Vector2(1f, rt.anchorMax.y); + // Negative width leaves room on both sides: the inset on the left, plus the same again + // on the right so text stops short of the arrow rather than touching it. + rt.sizeDelta = new Vector2(-(inset + 10f), rt.sizeDelta.y); + rt.anchoredPosition = new Vector2(inset * 0.5f, rt.anchoredPosition.y); + label.horizontalOverflow = HorizontalWrapMode.Overflow; + label.alignment = TextAnchor.MiddleLeft; + EditorUtility.SetDirty(label); + EditorUtility.SetDirty(rt); + } + + Debug.Log($"[BodyTrackingSetup] {path}: bodyModeDrop widened to {dropdownWidth}px, caption and items stretched"); + } + + private static void RelabelBodyModeOptions(UnityEngine.UI.Dropdown drop, string path) + { + var labels = new[] { "Off", "Upper Body (70)", "Full Body (84)" }; + if (drop.options.Count != labels.Length) + { + Debug.LogWarning($"[BodyTrackingSetup] {path}: bodyModeDrop has {drop.options.Count} " + + $"options, expected {labels.Length}; leaving labels alone"); + return; + } + + for (var i = 0; i < labels.Length; i++) + { + drop.options[i].text = labels[i]; + } + + drop.RefreshShownValue(); + EditorUtility.SetDirty(drop); + Debug.Log($"[BodyTrackingSetup] {path}: bodyModeDrop labels -> {string.Join(" / ", labels)}"); + } +} diff --git a/Assets/Editor/BodyTrackingSetup.cs.meta b/Assets/Editor/BodyTrackingSetup.cs.meta new file mode 100644 index 0000000..a5240c6 --- /dev/null +++ b/Assets/Editor/BodyTrackingSetup.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 38984cf7ea28d66e6920399c9e405b70 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Main.unity b/Assets/Main.unity index 325ca25..0109f21 100644 --- a/Assets/Main.unity +++ b/Assets/Main.unity @@ -507,6 +507,8 @@ MonoBehaviour: rightController: {fileID: 1882745283} leftControllerVisual: {fileID: 1280121364} rightControllerVisual: {fileID: 909864678} + body: {fileID: 533776542} + trackingSpace: {fileID: 827690208} --- !u!1 &25213884 GameObject: m_ObjectHideFlags: 0 @@ -2122,11 +2124,11 @@ RectTransform: m_Father: {fileID: 675691376} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0.5, y: 0.5} - m_AnchorMax: {x: 0.5, y: 0.5} - m_AnchoredPosition: {x: 0, y: 0} - m_SizeDelta: {x: 280.3, y: 30} - m_Pivot: {x: 0.5, y: 0.5} + m_AnchorMin: {x: 0, y: 0.5} + m_AnchorMax: {x: 0, y: 0.5} + m_AnchoredPosition: {x: -139, y: 0} + m_SizeDelta: {x: 265, y: 30} + m_Pivot: {x: 0, y: 0.5} --- !u!222 &125537320 CanvasRenderer: m_ObjectHideFlags: 0 @@ -2157,15 +2159,15 @@ MonoBehaviour: m_Calls: [] m_FontData: m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} - m_FontSize: 10 + m_FontSize: 11 m_FontStyle: 0 m_BestFit: 0 m_MinSize: 1 m_MaxSize: 14 - m_Alignment: 0 + m_Alignment: 3 m_AlignByGeometry: 0 m_RichText: 1 - m_HorizontalOverflow: 0 + m_HorizontalOverflow: 1 m_VerticalOverflow: 0 m_LineSpacing: 1 m_Text: @@ -3786,9 +3788,9 @@ RectTransform: m_Father: {fileID: 1022825376} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0, y: 1} - m_AnchorMax: {x: 0, y: 1} - m_AnchoredPosition: {x: -267, y: -20.7} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} m_SizeDelta: {x: 36.7, y: 23.4} m_Pivot: {x: 0, y: 0.5} --- !u!114 &209863748 @@ -6675,10 +6677,10 @@ RectTransform: m_Father: {fileID: 2032583195} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0.5, y: 0.5} - m_AnchorMax: {x: 0.5, y: 0.5} - m_AnchoredPosition: {x: 0, y: 0} - m_SizeDelta: {x: 45, y: 20} + m_AnchorMin: {x: 0, y: 0.5} + m_AnchorMax: {x: 1, y: 0.5} + m_AnchoredPosition: {x: 5, y: 0} + m_SizeDelta: {x: -20, y: 20} m_Pivot: {x: 0.5, y: 0.5} --- !u!114 &398004546 MonoBehaviour: @@ -6710,10 +6712,10 @@ MonoBehaviour: m_Alignment: 3 m_AlignByGeometry: 0 m_RichText: 1 - m_HorizontalOverflow: 0 + m_HorizontalOverflow: 1 m_VerticalOverflow: 0 m_LineSpacing: 1 - m_Text: None + m_Text: Off --- !u!222 &398004547 CanvasRenderer: m_ObjectHideFlags: 0 @@ -8721,6 +8723,7 @@ GameObject: - component: {fileID: 533776539} - component: {fileID: 533776538} - component: {fileID: 533776537} + - component: {fileID: 533776542} m_Layer: 0 m_Name: OVRCameraRig m_TagString: Untagged @@ -8815,7 +8818,7 @@ MonoBehaviour: launchSimultaneousHandsControllersOnStartup: 0 isInsightPassthroughEnabled: 1 shouldBoundaryVisibilityBeSuppressed: 0 - requestBodyTrackingPermissionOnStartup: 0 + requestBodyTrackingPermissionOnStartup: 1 requestFaceTrackingPermissionOnStartup: 0 requestEyeTrackingPermissionOnStartup: 0 requestScenePermissionOnStartup: 0 @@ -8884,6 +8887,19 @@ MonoBehaviour: installationRoutineCheckpoint: _installationRoutineId: _installationVariants: [] +--- !u!114 &533776542 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 533776536} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 274835e4e2feae04b87f0d000555f8a4, type: 3} + m_Name: + m_EditorClassIdentifier: + _providedSkeletonType: 0 --- !u!1 &534072469 GameObject: m_ObjectHideFlags: 0 @@ -13193,7 +13209,7 @@ GameObject: m_Icon: {fileID: 0} m_NavMeshLayer: 0 m_StaticEditorFlags: 0 - m_IsActive: 1 + m_IsActive: 0 --- !u!224 &748445603 RectTransform: m_ObjectHideFlags: 0 @@ -18190,9 +18206,9 @@ RectTransform: m_Father: {fileID: 1671116369} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0, y: 1} - m_AnchorMax: {x: 0, y: 1} - m_AnchoredPosition: {x: 67, y: -22.5} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} m_SizeDelta: {x: 461.3, y: 45} m_Pivot: {x: 0, y: 0.5} --- !u!114 &1022825377 @@ -18208,13 +18224,13 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: m_Padding: - m_Left: -267 + m_Left: 0 m_Right: 0 - m_Top: 9 - m_Bottom: 0 - m_ChildAlignment: 0 - m_Spacing: -623.4 - m_ChildForceExpandWidth: 1 + m_Top: 0 + m_Bottom: 8 + m_ChildAlignment: 3 + m_Spacing: 8 + m_ChildForceExpandWidth: 0 m_ChildForceExpandHeight: 0 m_ChildControlWidth: 0 m_ChildControlHeight: 0 @@ -21838,7 +21854,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 2609c54f376cffc4da1ab9401cc1a36f, type: 3} m_Name: m_EditorClassIdentifier: - _skeletonType: 4 + _skeletonType: 0 _updateRootPose: 0 _updateRootScale: 1 _enablePhysicsCapsules: 0 @@ -29472,7 +29488,7 @@ GameObject: m_Icon: {fileID: 0} m_NavMeshLayer: 0 m_StaticEditorFlags: 0 - m_IsActive: 0 + m_IsActive: 1 --- !u!224 &1671116369 RectTransform: m_ObjectHideFlags: 0 @@ -29489,10 +29505,10 @@ RectTransform: m_Father: {fileID: 753016364} m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0, y: 1} - m_AnchorMax: {x: 0, y: 1} - m_AnchoredPosition: {x: 528.3, y: -29.5} - m_SizeDelta: {x: 528.3, y: 45} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} m_Pivot: {x: 1, y: 0.5} --- !u!114 &1671116370 MonoBehaviour: @@ -29507,13 +29523,13 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: m_Padding: - m_Left: 67 + m_Left: 1 m_Right: 0 m_Top: 0 m_Bottom: 0 m_ChildAlignment: 0 - m_Spacing: -9.71 - m_ChildForceExpandWidth: 1 + m_Spacing: 0 + m_ChildForceExpandWidth: 0 m_ChildForceExpandHeight: 1 m_ChildControlWidth: 0 m_ChildControlHeight: 0 @@ -34970,7 +34986,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 2609c54f376cffc4da1ab9401cc1a36f, type: 3} m_Name: m_EditorClassIdentifier: - _skeletonType: 5 + _skeletonType: 1 _updateRootPose: 0 _updateRootScale: 1 _enablePhysicsCapsules: 0 @@ -35141,10 +35157,10 @@ RectTransform: m_Father: {fileID: 1022825376} m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0, y: 1} - m_AnchorMax: {x: 0, y: 1} - m_AnchoredPosition: {x: -196.20001, y: -21} - m_SizeDelta: {x: 70, y: 24} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 130, y: 24} m_Pivot: {x: 0.5, y: 0.5} --- !u!114 &2032583196 MonoBehaviour: @@ -35233,11 +35249,11 @@ MonoBehaviour: m_Value: 0 m_Options: m_Options: - - m_Text: None + - m_Text: Off m_Image: {fileID: 0} - - m_Text: Upper + - m_Text: Upper Body (70) m_Image: {fileID: 0} - - m_Text: Full + - m_Text: Full Body (84) m_Image: {fileID: 0} m_OnValueChanged: m_PersistentCalls: @@ -35365,8 +35381,8 @@ RectTransform: m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} - m_AnchoredPosition: {x: 10, y: -0.5} - m_SizeDelta: {x: -20, y: -3} + m_AnchoredPosition: {x: 11, y: -0.5} + m_SizeDelta: {x: -32, y: -3} m_Pivot: {x: 0.5, y: 0.5} --- !u!114 &2037206164 MonoBehaviour: @@ -35398,7 +35414,7 @@ MonoBehaviour: m_Alignment: 3 m_AlignByGeometry: 0 m_RichText: 1 - m_HorizontalOverflow: 0 + m_HorizontalOverflow: 1 m_VerticalOverflow: 0 m_LineSpacing: 1 m_Text: Option A diff --git a/Assets/Oculus/OculusProjectConfig.asset b/Assets/Oculus/OculusProjectConfig.asset index bad6c7a..e896ff1 100644 --- a/Assets/Oculus/OculusProjectConfig.asset +++ b/Assets/Oculus/OculusProjectConfig.asset @@ -21,7 +21,7 @@ MonoBehaviour: sharedAnchorSupport: 0 renderModelSupport: 0 trackedKeyboardSupport: 0 - bodyTrackingSupport: 0 + bodyTrackingSupport: 1 faceTrackingSupport: 0 eyeTrackingSupport: 0 virtualKeyboardSupport: 0 diff --git a/Assets/Resources/OculusRuntimeSettings.asset b/Assets/Resources/OculusRuntimeSettings.asset index cae0eeb..b3c88b1 100644 --- a/Assets/Resources/OculusRuntimeSettings.asset +++ b/Assets/Resources/OculusRuntimeSettings.asset @@ -18,5 +18,5 @@ MonoBehaviour: requestsAudioFaceTracking: 1 enableFaceTrackingVisemesOutput: 0 telemetryProjectGuid: 85b2eaab-0842-4d63-b652-5e31f2cdc409 - bodyTrackingFidelity: 1 + bodyTrackingFidelity: 2 bodyTrackingJointSet: 0 diff --git a/Assets/Scripts/Quest/QuestTrackingDataSource.cs b/Assets/Scripts/Quest/QuestTrackingDataSource.cs index 64bbc11..cfa5eb2 100644 --- a/Assets/Scripts/Quest/QuestTrackingDataSource.cs +++ b/Assets/Scripts/Quest/QuestTrackingDataSource.cs @@ -121,4 +121,141 @@ public void GetJoints(Handedness handedness, ref Pose[] poses) poses[i] = joints[i].GetPose(); } } + + /// + /// Body tracking (Meta Inside-Out Body Tracking, IOBT) source. + /// + /// Assign in the inspector, or leave null to auto-find at Start. Body tracking requires + /// OVRManager > Quest Features > Body Tracking Support = Supported and + /// Movement Tracking > Body Tracking Fidelity = High (Low is IK-only, not camera-measured). + /// + public OVRBody body; + + /// + /// The tracking space transform, i.e. the origin that IOBT joint coordinates are relative to. + /// + /// Despite the name, OVRCameraRig is not about cameras: it is the root of the whole XR tracking + /// space, and the camera is merely one of its children. Assign OVRCameraRig.trackingSpace here. + /// Only needed for body tracking, which is why the other pose sources above do without it: + /// arm posture is relative to the shoulders and independent of where the operator stands. + /// + public Transform trackingSpace; + + private void Start() + { + // Both are optional in the inspector so existing scenes keep working; fall back to a + // one-time scene scan rather than doing this per frame at the 90 Hz data rate. + if (body == null) + { + body = FindObjectOfType(); + } + + if (trackingSpace == null) + { + var rig = FindObjectOfType(); + trackingSpace = rig != null ? rig.trackingSpace : null; + } + + // Explicit, because both of these otherwise fail silently: body tracking simply produces + // nothing, with no error anywhere in logcat to indicate why. + if (body == null) + { + Debug.LogWarning("[QuestTrackingDataSource] no OVRBody in scene: body tracking will " + + "produce no data. Run BodyTrackingSetup.Configure."); + } + + if (trackingSpace == null) + { + Debug.LogWarning("[QuestTrackingDataSource] no trackingSpace: body joint poses will " + + "be sent without the tracking-space origin, so locomotion cannot be " + + "reconstructed downstream."); + } + } + + /// + /// True when body tracking has produced at least one valid frame. + /// + /// + /// Returns false whenever the headset is off the head, including hung around the neck. + /// + /// What fails is ovrp_GetBodyState4 returning false. Nothing in the managed SDK gates + /// on mount state -- OVRBody has no such reference -- so the decision is behind that + /// DllImport, in the closed-source native plugin or the OS tracking service. It cannot be + /// read, only measured, and the measurement is unambiguous: body data appears and disappears + /// on the same sample where sys.hmt.mounted flips, across repeated don/doff cycles, + /// while the app keeps XR focus, the cameras stay initialised and IOBT inference keeps + /// running on the DSP. This is neither power management nor the proximity sensor setting: + /// proximity_sensor_enabled was already 0 and made no difference, and the property + /// cannot be written without root, so there is nothing to work around here. + /// + /// Controller poses are NOT gated this way, which is why controller teleop survives being + /// hung round the neck and body tracking does not: sampling both against the property showed + /// body flipping in lockstep with it across three don/doff cycles while controller poses kept + /// moving for 40s+ off-head. Controllers carry their own IMU and are tracked optically by the + /// still-running cameras, whereas body has to be inferred from images of the operator, and + /// that inference is withheld -- not skipped -- while unmounted. + /// + /// The stay-awake keep-alive (stay_on_while_plugged_in=7, a plug-type bitmask of + /// AC|USB|wireless) only prevents the display sleeping so the CPU and TCP link survive. It is + /// a different failure: it was fully in effect during the measurements above. + /// + public bool IsBodyTrackingActive() + { + return body != null && body.BodyState != null; + } + + /// + /// Switches the joint set the runtime tracks and this component reads, e.g. 70-joint + /// UpperBody to 84-joint FullBody. Returns false if the runtime refused. + /// + /// + /// Both assignments are needed and neither is redundant, because the SDK reads the joint set + /// from two places: SetRequestedJointSet restarts the tracking session against + /// OVRRuntimeSettings, while GetBodyState4 is called with OVRBody's own + /// ProvidedSkeletonType. Setting only the first leaves the runtime producing 84 joints that + /// this component still asks for 70 of; setting only the second asks for joints the session + /// was never started to produce. The SDK does not reconcile them -- OVRBody.OnEnable only + /// logs a warning when they disagree. + /// + public bool SwitchJointSet(OVRPlugin.BodyJointSet jointSet) + { + if (body != null) + { + body.ProvidedSkeletonType = jointSet; + } + + var ok = OVRBody.SetRequestedJointSet(jointSet); + Debug.Log($"[QuestTrackingDataSource] joint set -> {jointSet} ({(ok ? "ok" : "REJECTED")})"); + return ok; + } + + /// + /// Gets the raw body tracking state, or null when unavailable. + /// + /// + /// Joint positions are expressed in tracking space, NOT world space, so they only describe + /// posture. Recovering where the operator stands requires the tracking space pose as well + /// (see ). + /// + public OVRPlugin.BodyState? GetBodyState() + { + return body != null ? body.BodyState : null; + } + + /// + /// Gets the tracking space pose in world coordinates. + /// + /// + /// Required to reconstruct root locomotion (walking around the room): joint coordinates are + /// tracking-space local, so posture alone cannot tell where the operator is. + /// + public Pose GetTrackingSpacePose() + { + if (trackingSpace == null) + { + return Pose.identity; + } + + return new Pose(trackingSpace.position, trackingSpace.rotation); + } } \ No newline at end of file diff --git a/Assets/Scripts/TrackingData.cs b/Assets/Scripts/TrackingData.cs index 0ca85c9..0025ceb 100644 --- a/Assets/Scripts/TrackingData.cs +++ b/Assets/Scripts/TrackingData.cs @@ -19,8 +19,8 @@ public class TrackingData public static bool HandTrackingOn { get; private set; } public static TrackingType TrackingTypeValue { get; private set; } - private JsonData _motionTrackingJson = new JsonData(); private JsonData _bodyTrackingJson = new JsonData(); + private JsonData _controllerDataJson = new JsonData(); private JsonData _leftControllerJson = new JsonData(); private JsonData _rightControllerJson = new JsonData(); @@ -45,6 +45,31 @@ public QuestTrackingDataSource questTrackingDataSource } } + /// + /// The scene's tracking data source, for callers that hold no TrackingData instance. + /// + /// + /// The UI needs the same body state that gets published, to show calibration progress. + /// TrackingData is instantiated by TcpHandler and not otherwise reachable, so rather than + /// have the UI run its own FindObjectOfType this exposes the one cached lookup. There is + /// only ever one source in the scene, so caching statically is safe. + /// + public static QuestTrackingDataSource SharedQuestTrackingDataSource + { + get + { + if (_sharedQuestTrackingDataSource == null) + { + _sharedQuestTrackingDataSource = + GameObject.FindObjectOfType(); + } + + return _sharedQuestTrackingDataSource; + } + } + + private static QuestTrackingDataSource _sharedQuestTrackingDataSource; + /// /// Sets whether head tracking is enabled /// @@ -73,7 +98,7 @@ public static void SetHandTrackingOn(bool on) } /// - /// Sets the current tracking type (None, Body, Motion) + /// Sets the current tracking type (None, Body, FullBody) /// /// The tracking type to set public static void SetTrackingType(TrackingType trackingType) @@ -138,12 +163,20 @@ public void Get(ref JsonData totalData) totalData.Remove("Hand"); } - // Remove PICO-specific tracking features for now - // Body and Motion tracking would need to be implemented with OpenXR equivalents - if (totalData.ContainsKey("Body")) - totalData.Remove("Body"); - if (totalData.ContainsKey("Motion")) - totalData.Remove("Motion"); + // Published under "BodyMeta" rather than "Body": the latter is a fixed 24-joint + // layout with per-joint velocity and acceleration, none of which Meta IOBT + // provides. See Docs/BodyMeta.md for the format. + if (JointSetOf(TrackingTypeValue) != null) + { + GetBodyMetaJsonData(); + totalData["BodyMeta"] = _bodyTrackingJson; + } + else + { + // totalData is reused across frames, so a stale key would keep being sent. + if (totalData.ContainsKey("BodyMeta")) + totalData.Remove("BodyMeta"); + } long nsTime = Utils.GetCurrentTimestamp(); totalData["timeStampNs"] = nsTime; @@ -308,6 +341,120 @@ private void GetHandJsonData() _handData["rightHand"] = _rightHandData; } + /// + /// Builds the BodyMeta JSON payload from Meta IOBT body tracking. + /// + /// + /// Every tracked joint is forwarded losslessly, tagged with its raw OVRPlugin.BoneId. + /// Sending the id rather than relying on array order makes the payload self-describing: + /// consumers resolve semantics against OVRPlugin.cs instead of guessing an index mapping. + /// This matters because the UpperBody and FullBody skeletons use different id layouts + /// (e.g. Body_LeftHandWrist = Body_Start+19 while Body_RightHandWrist = Body_Start+45), + /// so "jointSet" must be read before interpreting any id. + /// + /// Velocity/acceleration are absent by design: Meta's BodyJointLocation carries only + /// LocationFlags and Pose, unlike PICO's IMU trackers. Differentiate downstream if needed. + /// + private void GetBodyMetaJsonData() + { + var state = questTrackingDataSource.GetBodyState(); + var isActive = state != null && state.Value.JointLocations != null; + + _bodyTrackingJson["isActive"] = isActive ? 1U : 0U; + + // The tracking space pose is what makes root locomotion recoverable: joint coordinates + // are tracking-space local, so posture alone cannot say where the operator stands. + var trackingSpacePose = questTrackingDataSource.GetTrackingSpacePose(); + var tsPosition = trackingSpacePose.position; + var tsRotation = trackingSpacePose.rotation; + ConvertHandedness(ref tsPosition, ref tsRotation); + _bodyTrackingJson["trackingSpace"] = GetPoseStr(tsPosition, tsRotation); + + // JsonData objects are reused across frames rather than reallocated, matching the + // PICO client. At the 90 Hz send rate a fresh object per joint would mean ~70 + // allocations every frame, which is avoidable GC churn on a mobile SoC. + JsonData jointsJson; + if (_bodyTrackingJson.ContainsKey("joints")) + { + jointsJson = _bodyTrackingJson["joints"]; + } + else + { + jointsJson = new JsonData(); + jointsJson.SetJsonType(JsonType.Array); + _bodyTrackingJson["joints"] = jointsJson; + } + + if (!isActive) + { + _bodyTrackingJson["count"] = 0; + return; + } + + var bodyState = state.Value; + var joints = bodyState.JointLocations; + + // Data-quality metadata, passed through for the consumer to interpret rather than + // acted on here. Joints are published regardless of calib, matching Unity-Movement's + // sample scene, which drives its avatar throughout Calibrating with no visible difference + // once Valid arrives. Calibration lives in the runtime: it cannot be disabled or + // pre-seeded, and it restarts on every re-don, so gating on it would stall the stream + // for 30-60s each time for no gain. Calibrating means the runtime is still adjusting + // skeleton scale, not that there is no data. + _bodyTrackingJson["jointSet"] = bodyState.JointSet.ToString(); + _bodyTrackingJson["fidelity"] = bodyState.Fidelity.ToString(); + _bodyTrackingJson["calib"] = bodyState.CalibrationStatus.ToString(); + _bodyTrackingJson["confidence"] = bodyState.Confidence; + _bodyTrackingJson["count"] = joints.Length; + + // Drop entries the current joint set does not have. The array is reused across + // frames, and the loop below only overwrites the first joints.Length of them, so + // switching FullBody -> UpperBody would otherwise leave the 14 leg joints behind + // holding their last FullBody coordinates. A consumer sees an 84-entry array + // disagreeing with count=70, and the stale joints look like tracking that froze + // rather than a joint set that shrank. + // Cast for RemoveAt: LitJson implements it explicitly via IList. Removing by + // index rather than by value, since Remove(object) searches by equality and two + // joints sharing a pose would delete the wrong entry. + var jointsList = (System.Collections.IList)jointsJson; + while (jointsList.Count > joints.Length) + { + jointsList.RemoveAt(jointsList.Count - 1); + } + + for (var i = 0; i < joints.Length; i++) + { + var joint = joints[i]; + + // Two distinct conversions in sequence, not a redundant double flip: + // FromFlippedZ* : OVRPlugin (OpenXR, right-handed) -> Unity (left-handed) + // ConvertHandedness : Unity -> the wire convention shared with the PICO client + // The hand path only needs the second one because Oculus.Interaction hands its + // Pose over already converted; raw BodyState joints are pre-conversion. + var position = joint.Pose.Position.FromFlippedZVector3f(); + var rotation = joint.Pose.Orientation.FromFlippedZQuatf(); + + ConvertHandedness(ref position, ref rotation); + + JsonData jointJson; + if (i < jointsJson.Count) + { + jointJson = jointsJson[i]; + } + else + { + jointJson = new JsonData(); + jointsJson.Add(jointJson); + } + + jointJson["id"] = i; + jointJson["p"] = GetPoseStr(position, rotation); + // Occluded joints keep stale values, so validity must travel with the data + jointJson["v"] = joint.PositionValid ? 1U : 0U; + jointJson["vr"] = joint.OrientationValid ? 1U : 0U; + } + } + /// /// Gets hand tracking data for a specific hand using OVR hand tracking system /// @@ -504,10 +651,33 @@ public enum TrackingType { None = 0, - // Body and Motion tracking would require OpenXR extensions - // Keeping enum for compatibility but functionality removed + /// Meta IOBT upper body: 70 camera-derived joints. Body = 1, - Motion = 2 + + /// The 70 above plus 14 lower-body joints from Generative Legs. + /// + /// The legs are inferred from head and upper-body motion, not seen: the headset + /// cameras cannot observe them while it is worn. Consumers that need measured joints + /// only should stay on . + /// + FullBody = 2 + } + + /// + /// The joint set a mode publishes, or null when it publishes no body data. + /// + /// + /// One switch answers both "should we publish" and "which layout", because they are the + /// same question -- adding a further body mode means adding one case here. + /// + public static OVRPlugin.BodyJointSet? JointSetOf(TrackingType trackingType) + { + switch (trackingType) + { + case TrackingType.Body: return OVRPlugin.BodyJointSet.UpperBody; + case TrackingType.FullBody: return OVRPlugin.BodyJointSet.FullBody; + default: return null; + } } } } \ No newline at end of file diff --git a/Assets/Scripts/UI/LogWindow.cs b/Assets/Scripts/UI/LogWindow.cs index 9c78ebc..c30507f 100644 --- a/Assets/Scripts/UI/LogWindow.cs +++ b/Assets/Scripts/UI/LogWindow.cs @@ -1,4 +1,5 @@ using System.Collections; +using System.Collections.Generic; using TMPro; using UnityEngine; using UnityEngine.UI; @@ -13,11 +14,44 @@ public class LogWindow : MonoBehaviour public RectTransform rectTransform; + /// + /// Messages logged from non-main threads, drained in Update. + /// + /// + /// The socket callbacks (BeginConnect/BeginReceive) run on thread pool threads and log from + /// there. Touching Unity APIs off the main thread throws, and in ConnectCallback that throw is + /// swallowed by the surrounding catch, which then sets state to CONNECT_ERROR -- reporting a + /// failure for a connection that had in fact been established. + /// + private static readonly Queue PendingMessages = new Queue(); + + private const int MaxPendingMessages = 256; + private void Awake() { _instance = this; } + private void Update() + { + // Kept short: the lock is contended by socket threads on every log line. + while (true) + { + string message; + lock (PendingMessages) + { + if (PendingMessages.Count == 0) + { + return; + } + + message = PendingMessages.Dequeue(); + } + + AppendText(message); + } + } + private IEnumerator AutoScrollCoroutine() { LayoutRebuilder.ForceRebuildLayoutImmediate(scrollRect.content as RectTransform); @@ -57,13 +91,19 @@ public void AppendText(string message) private static void Message(string message) { - if (_instance != null) - { - _instance.AppendText(message); - } - else + // Callers include socket threads, so nothing here may touch Unity APIs directly: even + // reading _instance's implicit bool operator (the != null null-check on a MonoBehaviour) + // is a main-thread call. Queue unconditionally and let Update resolve the instance. + lock (PendingMessages) { - _instance = FindObjectOfType(); + // Bounded because Update only drains while a LogWindow instance is alive; without a + // cap, logging from socket threads before/after that would grow without limit. + if (PendingMessages.Count >= MaxPendingMessages) + { + PendingMessages.Dequeue(); + } + + PendingMessages.Enqueue(message); } } diff --git a/Assets/Scripts/UI/UIOperate.cs b/Assets/Scripts/UI/UIOperate.cs index 2d8aeb3..cf2dbc1 100644 --- a/Assets/Scripts/UI/UIOperate.cs +++ b/Assets/Scripts/UI/UIOperate.cs @@ -51,16 +51,18 @@ private void Awake() #endif // ReconnectBtn.gameObject.SetActive(false); - // bodyModeDrop.onValueChanged.AddListener(OnBodyModeDrop); + bodyModeDrop.onValueChanged.AddListener(OnBodyModeDrop); HeadTog.onValueChanged.AddListener(OnHeadTog); ControllerTog.onValueChanged.AddListener(OnControllerTog); HandTrackingTog.onValueChanged.AddListener(OnHandTrackingTog); SendTog.onValueChanged.AddListener(OnSendTog); Version.text = "v: " + Application.version; - // HighAccuracy.gameObject.SetActive(bodyModeDrop.value > 0); + // Meta IOBT has no separate high-accuracy runtime switch: fidelity is a project setting + // (OVRRuntimeSettings.BodyTrackingFidelity), not a per-session call like PICO's + // StartBodyTracking(mode). The toggle stays hidden so it cannot imply an inactive control. + HighAccuracy.gameObject.SetActive(false); NetshareTog.onValueChanged.AddListener(OnNetShareTog); - // HighAccuracy.onValueChanged.AddListener(OnHighAccuracy); ReconnectBtn.onClick.AddListener(OnReconnectBtn); //The shared network function is only available on B-end devices. NetshareTog.gameObject.SetActive(false); @@ -194,48 +196,20 @@ public void OnWriteIpBtn() private void OnBodyModeDrop(int index) { - // TODO - - // TrackingData.TrackingType tType = (TrackingData.TrackingType)bodyModeDrop.value; - // int res = 0; - // bool support = false; + TrackingData.TrackingType tType = (TrackingData.TrackingType)bodyModeDrop.value; - // TODO: body tracking in Quest - // MotionTrackerMode trackingMode = PXR_MotionTracking.GetMotionTrackerMode(); - // if (tType == TrackingData.TrackingType.Body) - // { - // if (trackingMode != MotionTrackerMode.BodyTracking) - // { - // res = PXR_MotionTracking.CheckMotionTrackerModeAndNumber(MotionTrackerMode.BodyTracking, - // MotionTrackerNum.TWO); - // } - // - // PXR_MotionTracking.GetBodyTrackingSupported(ref support); - // } - // else if (tType == TrackingData.TrackingType.Motion) - // { - // if (trackingMode != MotionTrackerMode.MotionTracking) - // { - // res = PXR_MotionTracking.CheckMotionTrackerModeAndNumber(MotionTrackerMode.MotionTracking, - // MotionTrackerNum.ONE); - // } - // - // support = true; - // } + // There is nothing to calibrate or pair (IOBT runs off the headset cameras), so the only + // precondition is that the device supports body tracking at all -- which covers both joint + // sets, since Generative Legs needs no hardware beyond what UpperBody already uses. Reject + // up front rather than letting the dropdown sit on a mode that will never produce data. + if (TrackingData.JointSetOf(tType) != null && !OVRPlugin.bodyTrackingSupported) + { + bodyModeDrop.SetValueWithoutNotify(0); + RefreshBodyInfo(); + return; + } - // if (!support || res != 0) - // { - // BodyInfo.text = "Tracker exception, please connect to calibrate tracker!"; - // BodyInfo.color = Color.red; - // - // bodyModeDrop.SetValueWithoutNotify(0); - // return; - // } - // - // BodyInfo.color = Color.white; - // BodyInfo.text = "Tracker detection is normal!"; - // - // UpdateBodyTracking(); + UpdateBodyTracking(); } @@ -347,55 +321,98 @@ private void OnSendTog(bool on) } } - private void OnHighAccuracy(bool on) - { - UpdateBodyTracking(); - } private void UpdateBodyTracking() { TrackingData.TrackingType tType = (TrackingData.TrackingType)bodyModeDrop.value; - HighAccuracy.gameObject.SetActive(bodyModeDrop.value > 0); Debug.Log("UpdateBodyTracking " + tType); + + // Quest has no external trackers; IOBT is inferred from the headset cameras alone. TrackNum.text = ""; - // TODO: Update for Quest - // // Set bone length - // BodyTrackingBoneLength boneLength = new BodyTrackingBoneLength(); - // if (bodyModeDrop.value <= 0) - // { - // int ret = PXR_MotionTracking.StopBodyTracking(); - // BodyInfo.text = "BodyTracking close"; - // } - // else - // { - // MotionTrackerConnectState state = new MotionTrackerConnectState(); - // PXR_MotionTracking.GetMotionTrackerConnectStateWithSN(ref state); - // // PXR_MotionTracking.GetMotionTrackerConnectStateWithSN(ref state); - // TrackNum.text = "TrackerNum:" + state.trackerSum; - // - // if (tType == TrackingData.TrackingType.Body) - // { - // BodyTrackingMode mode = BodyTrackingMode.BTM_FULL_BODY_LOW; - // if (HighAccuracy.isOn) - // { - // mode = BodyTrackingMode.BTM_FULL_BODY_HIGH; - // } - // - // // Enable full body motion capture default mode - // int ret = PXR_MotionTracking.StartBodyTracking(mode, boneLength); - // BodyInfo.text = "Start BodyTracking " + ret; - // Debug.Log(" UpdateBodyTracking :" + ret + " trackerSum:" + state.trackerSum); - // } - // else if (tType == TrackingData.TrackingType.Motion) - // { - // BodyInfo.text = "Start MotionTracking"; - // } - // } + + // Switching before SetTrackingType, so the first published frame already carries the new + // joint set: the runtime stops and restarts the tracking session, and a frame read in + // between would report the old layout with the new mode selected. + var jointSet = TrackingData.JointSetOf(tType); + if (jointSet != null) + { + var source = TrackingData.SharedQuestTrackingDataSource; + if (source != null) + { + source.SwitchJointSet(jointSet.Value); + } + } TrackingData.SetTrackingType(tType); + RefreshBodyInfo(); + } + + /// + /// Writes the live body tracking status line. + /// + /// + /// Called every frame rather than only on dropdown change. Calibration is the reason: it + /// starts out Calibrating and reaches Valid tens of seconds later, and joint scale is not + /// trustworthy until it does. A status line written once at selection time can never show + /// that transition, so the operator has no way to tell when the data became usable -- which + /// is exactly the question they have while standing there wearing the headset. + /// + private void RefreshBodyInfo() + { + if (TrackingData.JointSetOf(TrackingData.TrackingTypeValue) == null) + { + BodyInfo.color = Color.white; + BodyInfo.text = "Body tracking off"; + return; + } + + // The tracking session is requested by OVRBody on enable and restarted when the joint set + // changes; selecting a mode here is otherwise just a publish decision, so the status line + // reports whether the runtime can actually serve us. + if (!OVRPlugin.bodyTrackingSupported) + { + BodyInfo.color = Color.red; + BodyInfo.text = "Body tracking unsupported on this device"; + return; + } + + if (!OVRPlugin.bodyTrackingEnabled) + { + // The system-level toggle is the usual culprit and cannot be changed from here. + BodyInfo.color = Color.red; + BodyInfo.text = "Enable Settings > Movement Tracking > Body Tracking"; + return; + } + + var source = TrackingData.SharedQuestTrackingDataSource; + var state = source != null ? source.GetBodyState() : null; + if (state == null || state.Value.JointLocations == null) + { + // Normal whenever the headset is off the head, neck included: the system's mount + // detection gates body tracking, so hanging it round the neck stops the data even + // though controller teleop keeps working. See IsBodyTrackingActive for the evidence. + BodyInfo.color = Color.yellow; + BodyInfo.text = "No body data - put the headset on"; + return; + } + + var body = state.Value; + // Green once calibration reads Valid, but this is a readout and not a gate: joints are + // published throughout, the same way Unity-Movement's sample scene consumes them. Calibration + // runs inside the runtime with no way to switch it off, so waiting for green is optional -- + // it only means the runtime has stopped adjusting the skeleton's scale. + var calibrated = body.CalibrationStatus == OVRPlugin.BodyTrackingCalibrationState.Valid; + BodyInfo.color = calibrated ? Color.green : Color.yellow; + // Fidelity is omitted: it is a build-time project setting, so showing it every frame spends + // width on a constant. The joint count is what confirms a mode switch took effect -- 70 for + // Upper Body, 84 for Full Body -- and calibration and confidence are the two that move. + BodyInfo.text = $"Joints: {body.JointLocations.Length} " + + $"Calibration: {body.CalibrationStatus} " + + $"Confidence: {body.Confidence:F2}"; } private float _lastTime = 0; + private float _lastBodyInfoRefresh = 0; // Update is called once per frame void Update() @@ -409,6 +426,14 @@ void Update() } } + // Throttled: calibration state changes over tens of seconds, so rewriting the string at + // the 90 Hz frame rate would only cost a Text layout rebuild per frame for no gain. + if (Time.time - _lastBodyInfoRefresh > 0.2f) + { + _lastBodyInfoRefresh = Time.time; + RefreshBodyInfo(); + } + if (AcontrolerTog != null && AcontrolerTog.isOn) { // Use Input Actions only diff --git a/Docs/BodyMeta.md b/Docs/BodyMeta.md new file mode 100644 index 0000000..9f2b31c --- /dev/null +++ b/Docs/BodyMeta.md @@ -0,0 +1,144 @@ +# BodyMeta — Quest body tracking wire format + +Quest body tracking is published under its own top-level key, **`BodyMeta`**, rather than reusing +PICO's `Body`. This document explains why, and specifies the format for anyone writing a consumer. + +## Why a separate key + +| | PICO `Body` | Quest `BodyMeta` | +|---|---|---| +| Source | IMU motion trackers (external hardware) | Meta IOBT — the headset's own cameras | +| Joints | 24, fixed | 70 (`UpperBody`) or 84 (`FullBody`) | +| Per-joint data | pose + velocity + acceleration + per-joint IMU timestamp | pose + two validity flags | +| Calibration | app supplies `BodyTrackingBoneLength` | runs inside the runtime, cannot be pre-seeded | + +The two are not interchangeable. Reusing `Body` would mean either silently truncating 70 joints into +24 slots, or redefining a key that existing PICO consumers already parse — so Quest gets its own key +and old consumers are unaffected. A consumer that sees `BodyMeta` knows it is talking to a Quest and +gets the joint layout from the `jointSet` field rather than assuming one. + +Velocity and acceleration are absent because `OVRPlugin.BodyJointLocation` carries only +`LocationFlags` and `Pose` — there is nothing to report. Differentiate positions downstream if you +need rates. + +## Format + +`BodyMeta` is present while **Mode** is either **Upper Body (70)** or **Full Body (84)**; it is +removed from the packet when Mode is Off. `count` and `jointSet` tell the consumer which set is +live, and both change as soon as the operator switches mode — so read the layout per frame rather +than caching it at startup. + +```json +{ + "timeStampNs": 1785486843349566208, + "BodyMeta": { + "isActive": 1, + "count": 70, + "jointSet": "UpperBody", + "fidelity": "High", + "calib": "Valid", + "confidence": 1.0, + "trackingSpace": "0.000,0.000,0.000,0.000,0.000,0.000,1.000", + "joints": [ + { "id": 0, "p": "0.012,0.934,-0.048,0.001,0.707,0.002,0.707", "v": 1, "vr": 1 } + ] + } +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `isActive` | int | `1` when tracking is live. **Check this first** — see below. | +| `count` | int | Number of entries in `joints`. `0` when inactive. | +| `jointSet` | string | `UpperBody` (70) or `FullBody` (84). Selects the consumer's joint-id table. | +| `fidelity` | string | `High` = camera-measured IOBT; `Low` = IK-only inference. | +| `calib` | string | `Valid` / `Calibrating` / `Invalid`. Informational, not a gate — see below. | +| `confidence` | float | Runtime's overall confidence, `0.0`–`1.0`. | +| `trackingSpace` | string | Tracking space's own pose in the world frame, same 7-component format as `p`. | +| `joints[].id` | int | Index into the joint set. | +| `joints[].p` | string | `x,y,z,qx,qy,qz,qw` — position in metres, rotation as a quaternion. | +| `joints[].v` | int | `PositionValid`. `0` when the joint is occluded. | +| `joints[].vr` | int | `OrientationValid`. Fails independently of `v`. | + +`timeStampNs` is at the **top level**, not inside `BodyMeta`: every source in a frame shares one +timestamp, which is what makes headset pose and body joints alignable. + +Coordinates are in the same left-handed wire convention as the rest of the client's data, so a +consumer needs no Quest-specific transform. Walking is already included — the tracking space is +fixed to the room, so joint positions carry the operator's translation. + +## Two things consumers must know + +**Check `isActive` before reading `joints`.** When the headset leaves the head, tracking stops but +the `joints` array **keeps its last values** — the JSON objects are reused rather than cleared, and +`calib`/`count` go stale too. Nothing in the numbers themselves reveals this, so a consumer that +skips the flag will act on a frozen pose. + +Body tracking requires the headset actually worn; hanging it on the neck stops the data. The gate is +the system's mount detection, not power management: with the headset off, the cameras and IOBT +inference keep running but the runtime stops handing results to applications. Controller poses are +not gated this way, so controller teleop behaves differently here. + +**Do not wait for `calib == "Valid"`.** Joints stream complete from the first frame. Calibration runs +inside the runtime, refining the skeleton's *scale*; `Valid` only means it stopped refining. Measured +on a Quest 3: + +``` + t isActive calib count + 61.5 1 Calibrating 70 <- put on: immediately 70 joints + 73.9 1 Valid 70 <- 12 s later it turns Valid + 78.0 0 Valid 70 <- taken off: stale values, isActive says so +``` + +All 70 joints flowed for 12 s before `Valid` arrived, and Unity-Movement's sample scene likewise +drives its avatar throughout `Calibrating` with no visible change when it settles. Calibration +cannot be disabled or pre-seeded (`BodyTrackingCalibrationInfo` exposes only `BodyHeight`, and +`SuggestBodyTrackingCalibrationOverride` is a suggestion the runtime may ignore) and it restarts on +every re-don, so gating on it would stall the stream for 30–60 s each time for no gain. + +Skeleton proportions come from the runtime's defaults. Retargeting generally consumes angles and +directions, with absolute limb lengths normalised away; if you do need them, subtract joint +positions — measured across 69 bones, 62 varied by under 5 mm, with left/right symmetric to the +hundredth of a centimetre. + +## Joint layout + +Selected at runtime from the client's Mode dropdown. `FullBody` keeps `UpperBody`'s ids and +meanings and appends to them, so a consumer that only understands the first 70 keeps working. + +| id range | count | contents | set | +|---|---|---|---| +| `0 – 7` | 8 | Root, Hips, SpineLower, SpineMiddle, SpineUpper, Chest, Neck, Head | both | +| `8 – 17` | 10 | per side: Shoulder, Scapula, ArmUpper, ArmLower, HandWristTwist | both | +| `18 – 43` | 26 | left hand: Palm, Wrist, thumb (4) + 4 fingers (5 each) | both | +| `44 – 69` | 26 | right hand: same layout | both | +| `70 – 83` | 14 | per side: LegUpper, LegLower, FootAnkleTwist, FootAnkle, FootSubtalar, FootTransverse, FootBall | `FullBody` only | + +Note `id 44` is `RightHandPalm` — the right hand starts at 44, not 45. + +The lower-body 14 are **inferred, not measured**: the cameras cannot see the operator's legs while +the headset is worn, so Generative Legs predicts them from head and upper-body motion. They are a +different kind of data from `0 – 69` and should not be treated as ground truth. Both legs parent to +`Hips`, and each `*FootAnkleTwist` parents to its `LegLower` alongside the ankle rather than in the +chain, mirroring how `LeftHandWristTwist` sits beside `LeftHandWrist`. + +For joint names, use `BodyJointId` from `BodyPrimitives.cs`, **not** +`OVRPlugin.BoneId.ToString()`: `BoneId` overlays several skeletons on the same numeric values, so it +reports hand joint names for body joints. + +Parent indices are only available at runtime via `OVRPlugin.GetSkeleton2()`; there is no static table +in the SDK source. + +## Project settings this depends on + +Body tracking fails **silently** if any of these is wrong — no error, no log, just no data. +`Assets/Editor/BodyTrackingSetup.cs` applies them all and can be re-run at any time +(`Tools > Body Tracking > Configure Project Settings`). + +| Setting | Location | Required value | +|---|---|---| +| `bodyTrackingSupport` | `OculusProjectConfig` | `1` | +| `bodyTrackingFidelity` | `OculusRuntimeSettings` | `2` (High) — `1` silently degrades to IK-only | +| `bodyTrackingJointSet` | `OculusRuntimeSettings` | `0` (UpperBody) — the startup default only; the Mode dropdown switches it at runtime | +| `requestBodyTrackingPermissionOnStartup` | `OculusProjectConfig` | `true` — otherwise `com.oculus.permission.BODY_TRACKING` is never requested | +| `OVRBody` component | scene | present — nothing requests a tracking session without it | diff --git a/Docs/ui.png b/Docs/ui.png index e406e45..f1891b2 100644 Binary files a/Docs/ui.png and b/Docs/ui.png differ diff --git a/ProjectSettings/ProjectSettings.asset b/ProjectSettings/ProjectSettings.asset index 97b05f2..eaef81b 100644 --- a/ProjectSettings/ProjectSettings.asset +++ b/ProjectSettings/ProjectSettings.asset @@ -137,7 +137,7 @@ PlayerSettings: 16:10: 1 16:9: 1 Others: 1 - bundleVersion: 1.0.1 + bundleVersion: 1.0.2 preloadedAssets: - {fileID: 0} - {fileID: 0} diff --git a/README.md b/README.md index 361adef..b9f8dc7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ | Tracking - Head | Toggle On/Off to send out head 6 DoF pose | | Tracking - Controller | Toggle On/Off to parse VR controller's 6 DoF pose and button status in data stream | | Tracking - Hand | Toggle On/Off to parse hand tracking data in data stream | -| Tracking - Body Tracking | Body tracking for Quest. Coming soon ...... | +| Tracking - Body Tracking | Mode dropdown: Off / Upper Body (70) / Full Body (84). Streams body joints from the headset cameras, published as `BodyMeta` — see [Docs/BodyMeta.md](Docs/BodyMeta.md). Full Body adds 14 lower-body joints inferred by Generative Legs | | Tracking - Data & Control - Send | Toggle On/Off to sync above selected poses between XR device and robot PC | | Tracking - Data & Control - Switch w/ A Button | Toggle On/Off to rapid pause or resume sync with the right-hand controller button A | | Tracking - Status | Panel to show tracking related information | @@ -38,6 +38,10 @@ ## Feature list - **Pose sync between XR device and robot PC** Transmits pose data from the XR headset to the robot-side PC for robot teleoperation. +- **Body tracking on Quest (Meta IOBT)** + Streams body joints inferred from the headset's own cameras — no external trackers. Selectable + at runtime: 70 upper-body joints, or 84 with Generative Legs. + Wire format and consumer notes: [Docs/BodyMeta.md](Docs/BodyMeta.md). - **Local pose and stereo vision data collection** Synchronously records stereo vision and pose data collected from the XR headset, stored in the device's `/Download` directory. - **Remote stereo vision sync between two XR headsets**