diff --git a/.gitignore b/.gitignore index f3a6e2e..abfa354 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ UnrealEngineAPI/4.26/PoseAILiveLink4.26.zip PoseAILiveLink*.zip UnityAPI/PoseAI_Demo/* UnityAPI/PoseAI_UnityDemo/* +UnityAPI/PoseAI_UnityDemo1.4/* +UnityAPI/PoseAI_UnityDemo1.4 * UnityAPI.zip + diff --git a/README.md b/README.md index 7a64a11..dc4fc71 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ We currently host scripts and examples for Unity here on Github, and we will add ### Python demo -We provde a simple python example server which connects to the app and just prints the pose data to console. This can be used as a basis for incorporating Pose Camera in a variety of applications. The API is straightforward - the app looks for a UDP or TCP server on the IP address / Port number provided in Settings (or uses Bonjour/Zeroconf to scan the local network for bonjour enabled services). The app expects a "handshake" message in JSON format, which sets the desired configuration, and then begins streaming skeletal pose estimates live at low latency. The python example demonstrates the various options and format for the handshake. Skeletal rotations are streamed by the app in JSON format, quaternion form (scalar last, left-hand rule). We describe this in more detail: [here](StreamFormat.md) +We provde a simple python example server which connects to the app and just prints the pose data to console. This can be used as a basis for incorporating Pose Camera in a variety of applications. The API is straightforward - the app looks for a UDP or TCP server on the IP address / Port number provided in Settings (or uses Bonjour/Zeroconf to scan the local network for bonjour enabled services). The app expects a "handshake" message in JSON format, which sets the desired configuration, and then begins streaming skeletal pose estimates live at low latency. The python example demonstrates the various options and format for the handshake. Skeletal rotations are streamed by the app in JSON format, quaternion form (scalar last, left-hand rule). We describe this in more detail: [here](https://github.com/PoseAI/PoseCameraAPI/blob/main/FrameworkDocumentation/StreamFormat.md) ### Documentation diff --git a/UnityAPI/PoseAI_UnityDemo.zip b/UnityAPI/PoseAI_UnityDemo3.0.zip similarity index 94% rename from UnityAPI/PoseAI_UnityDemo.zip rename to UnityAPI/PoseAI_UnityDemo3.0.zip index 600e5c0..eba819d 100644 Binary files a/UnityAPI/PoseAI_UnityDemo.zip and b/UnityAPI/PoseAI_UnityDemo3.0.zip differ diff --git a/UnityAPI/UnityScripts/PoseAIAnimationSocket.cs b/UnityAPI/UnityScripts/PoseAIAnimationSocket.cs new file mode 100644 index 0000000..0e4a3b5 --- /dev/null +++ b/UnityAPI/UnityScripts/PoseAIAnimationSocket.cs @@ -0,0 +1,19 @@ +// Copyright 2023 Pose AI Ltd. All rights reserved + +using UnityEngine; + + +namespace PoseAI +{ + /* + * Add gameobject sockets to skeleton pieces and attach this component. + * Set the bone and then PoseAICharacterAnimator can detect and use for things like extra penetration tests. + */ + + public class PoseAIAnimationSocket : MonoBehaviour + { + [Tooltip("The BONE this socket is locally offset from")] + public HumanBodyBones bone; + + } +} diff --git a/UnityAPI/UnityScripts/PoseAICharacterAnimator.cs b/UnityAPI/UnityScripts/PoseAICharacterAnimator.cs index 06443f3..4e33597 100644 --- a/UnityAPI/UnityScripts/PoseAICharacterAnimator.cs +++ b/UnityAPI/UnityScripts/PoseAICharacterAnimator.cs @@ -1,9 +1,8 @@ -// Copyright 2021 Pose AI Ltd. All rights reserved +// Copyright 2021-2023 Pose AI Ltd. All rights reserved using UnityEngine; using System; using System.Collections.Generic; -using System.Reflection; namespace PoseAI @@ -16,48 +15,79 @@ namespace PoseAI [RequireComponent(typeof(Animator))] [RequireComponent(typeof(PoseAISource))] public class PoseAICharacterAnimator : MonoBehaviour - { + { [Tooltip("Animation weighting for Pose Camera vs normal animation (0.0-1.0)")] public float AnimationAlpha = 1.0f; + + [Header("Facial rig")] + [Tooltip("Facial mesh to apply blendshapes.")] + public SkinnedMeshRenderer FaceMesh; + + [Tooltip("Eye mesh transform for applying gaze direction blendshapes.")] + public Transform LeftEye; + + [Tooltip("Eye mesh transform for applying gaze direction blendshapes.")] + public Transform RightEye; + + + [Header("Motion control")] + + [Tooltip("Approx height of avatar. Scales lateral/vertical movement.")] + public float RigHeight = 1.8f; + + [Tooltip("Moves pelvis laterally/vertically based on player motion (if using full body animation)")] + public Vector3 ScaleRootMotion = new(1.0f, 1.0f, 1.0f); + + [Tooltip("Whether to keep avatar in contact with ground")] + public bool PinToFloor = true; + + + [Header("Configuration")] [Tooltip("Whether to only use upper body rotations")] public bool UseUpperBodyOnly = true; - - [Tooltip("Moves root for sideways movement in camera frame, if using full body animation")] - public bool MoveRootSideways = false; - - [Tooltip("Approx height of rig from ankle to head. Scales lateral/vertical movement.")] - public float RigHeight = 1.6f; - - [Tooltip("Use only if rig has a prefix before all bone names (i.e. mixamorig1: if the root is at mixamorig1:Hips)")] - public string JointNamePrefix = ""; - [Tooltip("Use to override the default rootname if the rig has a different root than standard")] - public string OverrideRootName = ""; + [Tooltip("Use to override the default hip/pelvis if the rig has a different hip/pelvis than standard")] + public string OverridePelvisName = ""; [Tooltip("Use only if you have set up a custom joint-by-joint remapping array (for non-standard rig configurations)")] public string Remapping = ""; + [Tooltip("Use extra joints not included in humanoid avatar (like twist), if found on rig, in LateUpdate")] + public bool UseExtraJoints = true; + + private Animator _animator; private PoseAISource _source; - private Transform _rootJoint; - private Vector3 _smoothedRootTranslation = Vector3.zero; + private Transform _pelvisJoint; + private Vector3 _smoothedRootTranslation = Vector3.zero; private float idleAlpha = 1.0f; - private List _jointNames = new List(); private List _remapping = null; - + private List _penetrationSockets = new(); private PoseAIRigBase _rigObj; - + private Vector3 _preIKPelvisLocalPosition; + private Dictionary _extraJoints = new(); + public void SetSource(PoseAISource source) { _source = source; if (_source != null) { _rigObj = source.GetRig(); - string rootName = OverrideRootName != "" ? OverrideRootName : JointNamePrefix + _rigObj.GetRootName() ; - _rootJoint = FindJointTransformRecursive(transform, rootName, skipToChildren: true); - if (_rootJoint == null) - Debug.Log("Did not find rootjoint in " + transform.name + ". Try overriding with correct name in editor."); + string rootName = OverridePelvisName != "" ? OverridePelvisName : _rigObj.GetRootName(); + _pelvisJoint = FindJointTransformRecursive(transform, rootName, skipToChildren: true, canBeEnding: false); + if (_pelvisJoint == null) + Debug.LogWarning("Did not find rootjoint in " + transform.name + ". Try overriding with correct name in editor."); + else + { + _penetrationSockets = new List(_pelvisJoint.GetComponentsInChildren()); + + // find if any extra joints match with rig joints for supplementary animation. + foreach (KeyValuePair entry in _rigObj.GetExtraBones()) + { + _extraJoints.Add(entry.Key, FindJointTransformRecursive(_pelvisJoint, entry.Value, true)); + } + } } } @@ -73,25 +103,30 @@ public void SetRemapping(string remapping) } else { - Debug.Log("Could not find an existing remapping named " + Remapping); + Debug.LogWarning("Could not find an existing remapping named " + Remapping); } } + } - private void Start(){ + + private void Start() + { _animator = GetComponent(); SetSource(GetComponent()); if (_source == null) - Debug.Log("Missing source for PoseAI CharacterAnimator in " + transform.root.gameObject.name.ToString()); + Debug.LogError("Missing source for PoseAI CharacterAnimator in " + transform.root.gameObject.name.ToString()); SetRemapping(Remapping); } - + void OnAnimatorIK() { + float useAlpha = AnimationAlpha * Mathf.Clamp(idleAlpha, 0.0f, 1.0f); if (useAlpha > 0.0f && CheckValidRig()) { - var bones = PoseAIRigBase.bones; + var bones = _rigObj.GetBones(); + _preIKPelvisLocalPosition = _animator.GetBoneTransform(bones[0]).transform.position - _animator.GetBoneTransform(bones[0]).transform.parent.transform.position; Tuple, List> rotationTuple = _rigObj.GetRotationsCameraSpace(); List validity = rotationTuple.Item1; List rotations = rotationTuple.Item2; @@ -99,8 +134,9 @@ void OnAnimatorIK() if (UseUpperBodyOnly) { - startAt = 9; - } else + startAt = 9; + } + else { if (_remapping != null) { @@ -108,57 +144,124 @@ void OnAnimatorIK() } else { - SetBone(bones[0], _rigObj.GetBaseRotation() * rotations[0], useAlpha); + SetBone(bones[0],_rigObj.GetBaseRotation() * rotations[0], useAlpha); } startAt = 1; } - + var parentIndices = _rigObj.GetParentIndices(); for (int j = startAt; j < rotations.Count; ++j) { - var p = PoseAIRigBase.parentIndices[j]; + var p = parentIndices[j]; var bone = bones[j]; + if (bone != HumanBodyBones.LastBone && validity[j] && validity[p]) { if (_remapping != null) { SetBone(bone, Quaternion.Inverse(rotations[p] * _remapping[p]) * rotations[j] * _remapping[j], useAlpha); - } else + } + else { SetBone(bone, Quaternion.Inverse(rotations[p]) * rotations[j], useAlpha); } - } } - } + } + } void LateUpdate() { - if (AnimationAlpha > 0.0f & !UseUpperBodyOnly && CheckValidRig()) + if (AnimationAlpha > 0.0f && !UseUpperBodyOnly && CheckValidRig()) AdjustRootLocation(); + float useAlpha = AnimationAlpha * Mathf.Clamp(idleAlpha, 0.0f, 1.0f); + if (UseExtraJoints && useAlpha > 0.0f && CheckValidRig()) + AnimateExtraJoints(); + } + + void AnimateExtraJoints() + { + Tuple, List> rotationTuple = _rigObj.GetRotationsCameraSpace(); + List validity = rotationTuple.Item1; + List rotations = rotationTuple.Item2; + var parentIndices = _rigObj.GetParentIndices(); + foreach (KeyValuePair entry in _extraJoints) + { + + var j = entry.Key; + if (UseUpperBodyOnly && j < 9) // note 9 may not be the number of lower joints in a custom schedule so this may need to be altered + continue; + var p = parentIndices[j]; + if (validity[j] && validity[p]) + { + var boneTransform = entry.Value; + if (_remapping != null) + { + boneTransform.localRotation = Quaternion.Slerp(boneTransform.localRotation, Quaternion.Inverse(rotations[p] * _remapping[p]) * rotations[j] * _remapping[j], AnimationAlpha); + } + else + { + boneTransform.localRotation = Quaternion.Slerp(boneTransform.localRotation, Quaternion.Inverse(rotations[p]) * rotations[j], AnimationAlpha); + } + } + } } void Update() { - if (CheckValidRig()) + UpdateFaceMesh(); + } + + + private void UpdateFaceMesh() + { + if (FaceMesh != null && _rigObj != null) { - // smoothly transition to (over 1 second) and away (over 2 seconds) from pose cam animations if torso is not visible. Clamp above 1 to provide a few milliseconds before fade begins - idleAlpha = Mathf.Clamp(idleAlpha + Time.deltaTime * ((_rigObj.visibility.isTorso) ? 1.0f : -0.5f ), 0.0f, 1.1f); + List blendshapes = PoseAIFaceBlendShapeReorder.ReorderBlendshapes(ref _rigObj.blendshapes); + for (int f = 0; f < blendshapes.Count; f++) + { + FaceMesh.SetBlendShapeWeight(f, blendshapes[f] * 100.0f); //scale by 100 as Unity uses 0-100 and we predict 0.0-1.0 + } + float kScaleEye = 20.0f; // +- degrees for movement, multiplied by blenshape difference ranges from -1 to 1. + + float DownUpL = _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookDownLeft] - + _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookUpLeft]; + + float DownUpR = _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookDownRight] - + _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookUpRight]; + + // averaging ensures eyes will move up/down together which is "normal" behavior. We still allow cross eyes (independent inout) + float DownUp = 0.5f * (DownUpL + DownUpR); + + if (LeftEye != null) + { + float InOut = _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookInLeft] - + _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookOutLeft]; + LeftEye.localRotation = Quaternion.Euler(DownUp * kScaleEye, InOut * kScaleEye, 0.0f); + } + if (RightEye != null) + { + float InOut = _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookInRight] - + _rigObj.blendshapes[(int)PoseAIFaceBlendShapeReorder.PoseAIFaceBlendShapes.EyeLookOutRight]; + + RightEye.localRotation = Quaternion.Euler(DownUp * kScaleEye, -InOut * kScaleEye, 0.0f); + } + } } private bool CheckValidRig() { - return (_source != null && + return (_source != null && _animator != null && _animator.runtimeAnimatorController != null && _rigObj != null && - _rootJoint != null && + _pelvisJoint != null && _rigObj.HasRigInfo()); } private void SetBone(HumanBodyBones bone, Quaternion target, float alpha) - { + { if (alpha < 1.0f) target = Quaternion.Slerp(_animator.GetBoneTransform(bone).localRotation, target, alpha); _animator.SetBoneLocalRotation(bone, target); @@ -172,55 +275,64 @@ private void SetBone(HumanBodyBones bone, Quaternion target, float alpha) */ private void AdjustRootLocation() { - var scaBody = _rigObj.GetBody().Scalars; - var vecBody = _rigObj.GetBody().Vectors; - var vis = _rigObj.visibility; - - var targetTranslation = new Vector3(0.0f, GetBonePenetration(), 0.0f); - if (vis.isTorso && scaBody.BodyHeight > 0.0f) - targetTranslation.x = vecBody.HipScreen[0] * RigHeight / scaBody.BodyHeight * AnimationAlpha * idleAlpha; - - // smooths translation - float alpha = Mathf.Clamp(Time.deltaTime * 4.0f, 0.1f, 1.0f); - _smoothedRootTranslation = Vector3.Lerp(_smoothedRootTranslation, targetTranslation, alpha); - _rootJoint.localPosition += _smoothedRootTranslation; + + float[] hip = new float[3]; + _rigObj.GetBody().Vectors.Hip.CopyTo(hip); + Vector3 targetTranslation = new(-hip[0] * ScaleRootMotion.x, hip[2] * ScaleRootMotion.y, hip[1] * ScaleRootMotion.z); + targetTranslation = targetTranslation * RigHeight + _preIKPelvisLocalPosition; + + //makes sure nw part of the character goes below the parent of the pelvis (root or player object) + float bone_penetration = GetBonePenetration(); + if (PinToFloor || bone_penetration > 0.0f) + targetTranslation.y = bone_penetration; + // could be used to additionally smooth translation, although our engine already does some + float alpha = 1.0f; // Mathf.Clamp(Time.deltaTime * 10.0f, 0.1f, 1.0f); + _smoothedRootTranslation = Vector3.Lerp(_smoothedRootTranslation, targetTranslation, alpha); + _pelvisJoint.localPosition += _smoothedRootTranslation; + } /* calculates lowest Y relative to component space. */ private float GetBonePenetration() { float minY = float.MaxValue; - MethodInfo methodGetAxisLength = typeof(Avatar).GetMethod("GetAxisLength", BindingFlags.Instance | BindingFlags.NonPublic); - if (methodGetAxisLength != null) - { - float ltoes = (float)methodGetAxisLength.Invoke(_animator.avatar, new object[] { HumanBodyBones.LeftToes }); - float rtoes = (float)methodGetAxisLength.Invoke(_animator.avatar, new object[] { HumanBodyBones.RightToes }); - var lt = _animator.GetBoneTransform(HumanBodyBones.LeftToes); - var rt = _animator.GetBoneTransform(HumanBodyBones.RightToes); - - minY = Mathf.Min( - lt.position.y - (lt.rotation * new Vector3(0, ltoes, 0)).y, - rt.position.y - (rt.rotation * new Vector3(0, rtoes, 0)).y - ); - } - foreach (var bone in PoseAIRigBase.bones) + foreach (var bone in _rigObj.GetBones()) { if (bone != HumanBodyBones.LastBone) { - minY = Mathf.Min(minY, _animator.GetBoneTransform(bone).position.y); + var bt = _animator.GetBoneTransform(bone); + if (bt != null) + { + minY = Mathf.Min(minY, bt.position.y); + } + } + } + // also checks any penetration sockets added to avatar for better foot placement + foreach (var socket in _penetrationSockets) + { + var bt = _animator.GetBoneTransform(socket.bone); + if (bt != null) + { + minY = Mathf.Min(minY, bt.TransformPoint(socket.transform.localPosition).y); } + } - float componentY = _rootJoint.parent.position.y; + + float componentY = _pelvisJoint.parent.position.y; return componentY - minY; } - private static Transform FindJointTransformRecursive(Transform parent, string name, bool skipToChildren = false) + private static Transform FindJointTransformRecursive(Transform parent, string name, bool skipToChildren = false, bool canBeEnding=true) { if (!skipToChildren && string.Equals(parent.name, name, StringComparison.InvariantCultureIgnoreCase)) { return parent; } + if (!skipToChildren && canBeEnding && parent.name.EndsWith(name, StringComparison.InvariantCultureIgnoreCase)) + { + return parent; + } foreach (Transform child in parent) { diff --git a/UnityAPI/UnityScripts/PoseAIConfig.cs b/UnityAPI/UnityScripts/PoseAIConfig.cs index 2d2f432..db1f37c 100644 --- a/UnityAPI/UnityScripts/PoseAIConfig.cs +++ b/UnityAPI/UnityScripts/PoseAIConfig.cs @@ -15,15 +15,30 @@ public static class PoseAIConfig } - public enum PoseAI_Modes { Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly }; - public enum PoseAI_Rigs { Unity, UE4, Mixamo }; + public enum PoseAI_Modes { Room, SeatedAtDesk, Portrait, RoomBodyOnly, PortraitBodyOnly }; + public enum PoseAI_Rigs { Unity, UE4, Mixamo, MixamoAlt, MetaHuman }; public enum PoseAI_Gestures {None, Swipe1, Swipe2, Swipe3, Swipe4, Swipe6, Swipe7, Swipe8, Swipe9, FlapLateral, Flap, WaxOn, WaxOff, OverheadClapSmall, OverheadClap, ReverseOverheadClap}; public enum TouchState { Begun, Touching, Ended, Cancelled}; + + public enum PoseAI_CameraQuality {VGA, _360p, _720p, _1080p, QHD, _4K} + public static class PoseAI_Methods { - + public static Resolution Resolution(PoseAI_CameraQuality quality) + { + switch (quality) + { + case PoseAI_CameraQuality.VGA: return new Resolution { width= 480, height = 640 }; + case PoseAI_CameraQuality._360p: return new Resolution { width = 360, height = 640 }; + case PoseAI_CameraQuality._720p: return new Resolution { width = 1280, height = 720 }; + case PoseAI_CameraQuality._1080p: return new Resolution { width = 1920, height = 1080 }; + case PoseAI_CameraQuality.QHD: return new Resolution { width = 2560, height = 1440 }; + case PoseAI_CameraQuality._4K: return new Resolution { width = 3840, height = 2160 }; + default: return new Resolution { width = 480, height = 640 }; + } + } public static PoseAI_Gestures Gesture(this uint s1) { switch (s1) @@ -49,9 +64,9 @@ public static PoseAI_Gestures Gesture(this uint s1) } - public static bool IsDesktop(PoseAI_Modes mode) => mode switch + public static bool IsSeatedAtDesk(PoseAI_Modes mode) => mode switch { - PoseAI_Modes.Desktop => true, + PoseAI_Modes.SeatedAtDesk => true, _ => false }; } diff --git a/UnityAPI/UnityScripts/PoseAIFaceOrder.cs b/UnityAPI/UnityScripts/PoseAIFaceOrder.cs new file mode 100644 index 0000000..370a2e2 --- /dev/null +++ b/UnityAPI/UnityScripts/PoseAIFaceOrder.cs @@ -0,0 +1,149 @@ +// Copyright 2023 Pose AI Ltd. All rights reserved + +using System.Collections.Generic; +using UnityEngine; + +namespace PoseAI +{ + //specify your rig's blendshape order here, as Unity sets them by index order rather than by name + public class PoseAIFaceBlendShapeReorder + { + static public List MeshOrder = new List { + PoseAIFaceBlendShapes.BrowOuterUpLeft, + PoseAIFaceBlendShapes.BrowOuterUpRight, + PoseAIFaceBlendShapes.EyeSquintLeft, + PoseAIFaceBlendShapes.EyeSquintRight, + PoseAIFaceBlendShapes.EyeLookInLeft, + PoseAIFaceBlendShapes.EyeLookOutLeft, + PoseAIFaceBlendShapes.EyeLookInRight, + PoseAIFaceBlendShapes.EyeLookOutRight, + PoseAIFaceBlendShapes.EyeLookUpLeft, + PoseAIFaceBlendShapes.EyeLookUpRight, + PoseAIFaceBlendShapes.EyeLookDownLeft, + PoseAIFaceBlendShapes.EyeLookDownRight, + PoseAIFaceBlendShapes.CheekPuff, + PoseAIFaceBlendShapes.CheekSquintLeft, + PoseAIFaceBlendShapes.CheekSquintRight, + PoseAIFaceBlendShapes.NoseSneerLeft, + PoseAIFaceBlendShapes.NoseSneerRight, + PoseAIFaceBlendShapes.MouthLeft, + PoseAIFaceBlendShapes.MouthRight, + PoseAIFaceBlendShapes.MouthPucker, + PoseAIFaceBlendShapes.MouthFunnel, + PoseAIFaceBlendShapes.MouthSmileLeft, + PoseAIFaceBlendShapes.MouthSmileRight, + PoseAIFaceBlendShapes.MouthFrownLeft, + PoseAIFaceBlendShapes.MouthFrownRight, + PoseAIFaceBlendShapes.MouthDimpleLeft, + PoseAIFaceBlendShapes.MouthDimpleRight, + PoseAIFaceBlendShapes.MouthPressLeft, + PoseAIFaceBlendShapes.MouthPressRight, + PoseAIFaceBlendShapes.MouthShrugLower, + PoseAIFaceBlendShapes.MouthShrugUpper, + PoseAIFaceBlendShapes.MouthStretchLeft, + PoseAIFaceBlendShapes.MouthStretchRight, + PoseAIFaceBlendShapes.MouthUpperUpLeft, + PoseAIFaceBlendShapes.MouthUpperUpRight, + PoseAIFaceBlendShapes.MouthLowerDownLeft, + PoseAIFaceBlendShapes.MouthLowerDownRight, + PoseAIFaceBlendShapes.MouthRollUpper, + PoseAIFaceBlendShapes.MouthRollLower, + PoseAIFaceBlendShapes.MouthClose, + PoseAIFaceBlendShapes.JawForward, + PoseAIFaceBlendShapes.JawOpen, + PoseAIFaceBlendShapes.JawLeft, + PoseAIFaceBlendShapes.JawRight, + PoseAIFaceBlendShapes.BrowInnerUp, + PoseAIFaceBlendShapes.EyeBlinkRight, // odd that Unity sample mesh puts right before left + PoseAIFaceBlendShapes.EyeBlinkLeft, + PoseAIFaceBlendShapes.BrowDownLeft, + PoseAIFaceBlendShapes.BrowDownRight, + PoseAIFaceBlendShapes.EyeWideRight, // odd that Unity sample mesh puts right before left + PoseAIFaceBlendShapes.EyeWideLeft, + PoseAIFaceBlendShapes.JawOpen, + PoseAIFaceBlendShapes.JawForward, + PoseAIFaceBlendShapes.JawLeft, + PoseAIFaceBlendShapes.JawRight, + PoseAIFaceBlendShapes.TongueOut + }; + + static public List ReorderBlendshapes(ref List default_blendshapes) + { + List reordered_blendshapes = new List(); + foreach(var shape in MeshOrder) + { + reordered_blendshapes.Add(default_blendshapes[(int)shape]); + } + return reordered_blendshapes; + } + + + + public enum PoseAIFaceBlendShapes : int + { + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut + }; + + + } +} diff --git a/UnityAPI/UnityScripts/PoseAIJson.cs b/UnityAPI/UnityScripts/PoseAIJson.cs index a3d14f9..5fcfa00 100644 --- a/UnityAPI/UnityScripts/PoseAIJson.cs +++ b/UnityAPI/UnityScripts/PoseAIJson.cs @@ -1,4 +1,4 @@ -// Copyright 2021 Pose AI Ltd. All rights reserved +// Copyright 2021-2023 Pose AI Ltd. All rights reserved using UnityEngine; using System.Collections.Generic; @@ -10,6 +10,7 @@ namespace PoseAI { + public static class PoseAIRigFactory { public static PoseAIRigBase SelectRig(PoseAI_Rigs rigType) => rigType switch @@ -17,6 +18,8 @@ public static class PoseAIRigFactory PoseAI_Rigs.Unity => new PoseAIRigUnity(), PoseAI_Rigs.UE4 => new PoseAIRigUE4(), PoseAI_Rigs.Mixamo => new PoseAIRigMixamo(), + PoseAI_Rigs.MixamoAlt => new PoseAIRigMixamo(), + PoseAI_Rigs.MetaHuman => new PoseAIRigMetaHuman(), _ => new PoseAIRigUnity() }; } @@ -32,6 +35,7 @@ public class Handshake public string rig; public string mode; public string mirror; + public string face; public int syncFPS; public int cameraFPS; @@ -40,10 +44,13 @@ public class Handshake public string signature = "Your App Signature"; // 0 for verbose JSON, 1 for more compact packets. +#if VERBOSE_PACKET + public int packetFormat = 0; +#else public int packetFormat = 1; - - //this is the version of our AI. version 1 is our original model during beta and release. version 2 was released in March 2022 and is currently the default. - public int modelVersion = 2; +#endif + //this is the version of our AI. version 3 is current as of June 2023. + public int modelVersion = 3; } public Handshake HANDSHAKE = new Handshake(); @@ -60,19 +67,19 @@ public class Config //public double echoServerTimestamp = ; - public static PoseAIHandshake Factory(PoseAI_Modes mode, PoseAI_Rigs rig, bool mirror, int syncFPS, int cameraFPS) + public static PoseAIHandshake Factory(PoseAI_Modes mode, PoseAI_Rigs rig, bool mirror, bool face, int syncFPS, int cameraFPS) { PoseAIHandshake handshake = new PoseAIHandshake(); handshake.HANDSHAKE.rig = rig.ToString(); - handshake.HANDSHAKE.mode = mode.ToString(); + handshake.HANDSHAKE.mode = mode==PoseAI_Modes.SeatedAtDesk ? "Desktop" : mode.ToString(); //in process of migrating name handshake.HANDSHAKE.mirror = mirror ? "YES" : "NO"; + handshake.HANDSHAKE.face = face ? "YES" : "NO"; handshake.HANDSHAKE.cameraFPS = Mathf.Max(cameraFPS, PoseAIConfig.MIN_CAMERA_FPS); - handshake.HANDSHAKE.syncFPS = Mathf.Max(handshake.HANDSHAKE.cameraFPS, syncFPS); + handshake.HANDSHAKE.syncFPS = 0; //syncFPS is being depreciated. setting to zero to ensure all versions of engine don't use return handshake; } } - [System.Serializable] public abstract class PoseAIRigBase { @@ -91,6 +98,16 @@ public abstract class PoseAIRigBase // packet format. 0 is verbose, 1 is compressed public int PF; + // face blendshapes if available. This will only work if PF=1 (compressed). For Verbose need to change this to a List. May to resolve in future release +#if VERBOSE_PACKET + public List Face = new(52); +#else + public string Face = ""; +#endif + + + public List blendshapes = new List(new float[52]); + //receives compact touch updates from app, and adds them as vector 2s to Queue for game logic processing. Values are relative to phone not user (use orientation to rotate as appropriate) public string TouchState = ""; public string Touches = ""; @@ -110,7 +127,7 @@ public abstract class PoseAIRigBase public int lowerBodyNumOfJoints = 8; // this is the order the PoseCamera sends the joints. - public static readonly List bones = new List{ + private static readonly List bones = new List{ HumanBodyBones.Hips, HumanBodyBones.RightUpperLeg, HumanBodyBones.RightLowerLeg, @@ -166,11 +183,21 @@ public abstract class PoseAIRigBase HumanBodyBones.RightThumbIntermediate, HumanBodyBones.RightThumbDistal }; + public virtual List GetBones() { + return bones; + } + public virtual Dictionary GetExtraBones() { return new() { }; } + - public static readonly List parentIndices = new List { - 0,0,1,2,3,0,5,6,7,0,9,10,11,12,11,14,15,11,17,18,16,0,20,22,23,20,25,26,20,28,29,20,31,32,20,34,35,19,0,37,39,40,37,42,43,37,45,46,37,48,49,37,51,52, + public static readonly List genericParentIndices = new() { + 0,0,1,2,3,0,5,6,7,0,9,10,11,12,11,14,15,11,17,18,16,16,20,22,23,20,25,26,20,28,29,20,31,32,20,34,35,19,19,37,39,40,37,42,43,37,45,46,37,48,49,37,51,52, }; + public virtual List GetParentIndices() + { + return genericParentIndices; + } + protected internal List> rotationData; protected internal List rotationNames; protected internal int numOfLeftHandJoints; @@ -217,13 +244,18 @@ public bool OverwriteFromJSON(string jsonString) { GetBody().Scalars.ProcessCompact(ref GetBody().ScaA); GetBody().Vectors.ProcessCompact(ref GetBody().VecA); - GetBody().Vectors.ProcessCompact(ref GetBody().VecA); GetBody().Events.ProcessCompact(ref GetBody().EveA); visibility.ProcessCompact(ref GetBody().VisA); - } else + GetLeftHand().ProcessCompact(); + GetRightHand().ProcessCompact(); + + + } + else { visibility.ProcessVerbose(ref GetBody().Scalars); } + ProcessFace(); ProcessTouches(); return true; } @@ -337,6 +369,21 @@ private void AppendCompactRotations(ref string compactString, ref List 1) { @@ -419,6 +466,9 @@ public class VectorsBody public List HipLean = new List { 0.0f, 0.0f }; public List HandIkL = new List { 0.0f, 0.0f, 0.0f }; public List HandIkR = new List { 0.0f, 0.0f, 0.0f }; + public List Hip = new List { 0.0f, 0.0f, 0.0f }; + public List FootIkL = new List { 0.0f, 0.0f, 0.0f }; + public List FootIkR = new List { 0.0f, 0.0f, 0.0f }; public void ProcessCompact(ref string compactString) @@ -431,12 +481,12 @@ public void ProcessCompact(ref string compactString) ChestScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[8], compactString[9]); ChestScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[10], compactString[11]); if (compactString.Length < 24) return; - HandIkL[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[12], compactString[13]) * 4.0f; - HandIkL[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[14], compactString[15]) * 4.0f; - HandIkL[2] = PoseAI_Decoder.FixedB64pairToFloat(compactString[16], compactString[17]) * 4.0f; - HandIkR[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[18], compactString[19]) * 4.0f; - HandIkR[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[20], compactString[21]) * 4.0f; - HandIkR[2] = PoseAI_Decoder.FixedB64pairToFloat(compactString[22], compactString[23]) * 4.0f; + PoseAI_Decoder.FStringFixed12ToFloat(compactString.Substring(12, 6), ref HandIkL, 4.0f, true); + PoseAI_Decoder.FStringFixed12ToFloat(compactString.Substring(18, 6), ref HandIkR, 4.0f, true); + if (compactString.Length < 42) return; + PoseAI_Decoder.FStringFixed12ToFloat(compactString.Substring(24, 6), ref Hip, 4.0f, true); + PoseAI_Decoder.FStringFixed12ToFloat(compactString.Substring(30, 6), ref FootIkL, 4.0f, true); + PoseAI_Decoder.FStringFixed12ToFloat(compactString.Substring(36, 6), ref FootIkR, 4.0f, true); } } @@ -450,6 +500,7 @@ public class ScalarsBody public float VisArmR = 0.0f; public float VisLegL = 0.0f; public float VisLegR = 0.0f; + public float VisFace = 0.0f; /** location of left hand relative to body in broad zones */ @@ -510,8 +561,9 @@ public class VisibilityFlags public bool isRightArm = false; public bool isLeftLeg = false; public bool isRightLeg = false; + public bool isFace = false; + - public bool HasChanged() { return hasChanged; } public void ProcessVerbose(ref ScalarsBody bodyVerbose) { @@ -520,7 +572,8 @@ public void ProcessVerbose(ref ScalarsBody bodyVerbose) SetAndCheckForChange(bodyVerbose.VisLegL > 0.5f, ref isLeftLeg, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisLegR > 0.5f, ref isRightLeg, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisArmL > 0.5f, ref isLeftArm, ref hasChanged); - SetAndCheckForChange(bodyVerbose.VisArmR > 0.5f, ref isRightArm, ref hasChanged); + SetAndCheckForChange(bodyVerbose.VisArmR > 0.5f, ref isRightArm, ref hasChanged); + SetAndCheckForChange(bodyVerbose.VisFace > 0.5f, ref isFace, ref hasChanged); } public void ProcessCompact(ref string visString) @@ -531,6 +584,8 @@ public void ProcessCompact(ref string visString) SetAndCheckForChange(visString[2] != '0', ref isRightLeg, ref hasChanged); SetAndCheckForChange(visString[3] != '0', ref isLeftArm, ref hasChanged); SetAndCheckForChange(visString[4] != '0', ref isRightArm, ref hasChanged); + if (visString.Length>5) + SetAndCheckForChange(visString[5] != '0', ref isFace, ref hasChanged); } private bool hasChanged = false; @@ -656,6 +711,28 @@ public abstract class BaseHandWrapper // compressed format field public string RotA = ""; + + public string Point = ""; + + + // fist = 0.0, fully extended fingers = 1.0, can vary depending on number of straight fingers. + public float Open = 0.5f; + + public void ProcessCompact() + { + if (Point.Length >= 4) + { + Vectors.PointScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(Point[0], Point[1]); + Vectors.PointScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(Point[2], Point[3]); + } + if (Point.Length >= 8) + { + Vectors.ThumbScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(Point[4], Point[5]); + Vectors.ThumbScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(Point[6], Point[7]); + } + } + + } [System.Serializable] @@ -666,14 +743,9 @@ public class ScalarsHand { } public class VectorsHand { public List PointScreen = new List { 0.0f, 0.0f }; - public void ProcessCompact(ref string compactString) - { - if (compactString.Length < 4) return; - PointScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[0], compactString[1]); - PointScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[2], compactString[3]); - } + public List ThumbScreen = new List { 0.0f, 0.0f }; } -} +} \ No newline at end of file diff --git a/UnityAPI/UnityScripts/PoseAIRigMetaHuman.cs b/UnityAPI/UnityScripts/PoseAIRigMetaHuman.cs new file mode 100644 index 0000000..1f67f3a --- /dev/null +++ b/UnityAPI/UnityScripts/PoseAIRigMetaHuman.cs @@ -0,0 +1,252 @@ +// Copyright 2023 Pose AI Ltd. All rights reserved + + +// Important: Importing a MetaHumans avatar into Unity is a violation of Epic Games licensing permissions, and is not a permitted use of this script. +// We only provide this script for 3rd PARTY (non Epic Games) rigs which are formated to use the same skeleton naming convention and joint orientations as the MetaHuman template + + +using UnityEngine; +using System.Collections.Generic; + +// +namespace PoseAI +{ + [System.Serializable] + public class PoseAIRigMetaHuman : PoseAIRigBase + { + + // set to 180 degree rotation around Z based on how a test Unreal rig imported + public PoseAIRigMetaHuman():base(new Quaternion(0.0f, 0.7f, 0.7f, 0.0f)){} + + [System.Serializable] + public class BodyWrapper : BaseBodyWrapper + { + [System.Serializable] + public class RotationsWrapper : BaseRotationsWrapper + { + // Rig specific field names go here + public List pelvis; + public List thigh_r; + public List calf_r; + public List foot_r; + public List ball_r; + public List thigh_l; + public List calf_l; + public List foot_l; + public List ball_l; + public List spine_01; + public List spine_02; + public List spine_03; + public List spine_04; + public List spine_05; + public List neck_01; + public List neck_02; + public List head; + public List clavicle_l; + public List upperarm_l; + public List lowerarm_l; + public List clavicle_r; + public List upperarm_r; + public List lowerarm_r; + } + public RotationsWrapper Rotations = new RotationsWrapper(); + } + + [System.Serializable] + public class LeftHandWrapper : BaseHandWrapper + { + [System.Serializable] + public class RotationsWrapper : BaseRotationsWrapper + { + // Rig specific field names go here + public List hand_l; + public List lowerarm_twist_01_l; + public List lowerarm_twist_02_l; + public List index_metacarpal_l; + public List index_01_l; + public List index_02_l; + public List index_03_l; + public List middle_carpal_l; + public List middle_01_l; + public List middle_02_l; + public List middle_03_l; + public List ring_metacarpal_l; + public List ring_01_l; + public List ring_02_l; + public List ring_03_l; + public List pinky_metacarpal_l; + public List pinky_01_l; + public List pinky_02_l; + public List pinky_03_l; + public List thumb_01_l; + public List thumb_02_l; + public List thumb_03_l; + } + public RotationsWrapper Rotations = new RotationsWrapper(); + + } + + [System.Serializable] + public class RightHandWrapper : BaseHandWrapper + { + [System.Serializable] + public class RotationsWrapper : BaseRotationsWrapper + { + // Rig specific field names go here + public List hand_r; + public List lowerarm_twist_01_r; + public List lowerarm_twist_02_r; + public List index_metacarpal_r; + public List index_01_r; + public List index_02_r; + public List index_03_r; + public List middle_carpal_r; + public List middle_01_r; + public List middle_02_r; + public List middle_03_r; + public List ring_metacarpal_r; + public List ring_01_r; + public List ring_02_r; + public List ring_03_r; + public List pinky_metacarpal_r; + public List pinky_01_r; + public List pinky_02_r; + public List pinky_03_r; + public List thumb_01_r; + public List thumb_02_r; + public List thumb_03_r; + } + public RotationsWrapper Rotations = new RotationsWrapper(); + } + + public BodyWrapper Body = new BodyWrapper(); + public LeftHandWrapper LeftHand = new LeftHandWrapper(); + public RightHandWrapper RightHand = new RightHandWrapper(); + + protected internal override void BuildListsFromIntrospection() + { + Body.Rotations.AddFields(rotationData, rotationNames); + LeftHand.Rotations.AddFields(rotationData, rotationNames); + RightHand.Rotations.AddFields(rotationData, rotationNames); + numOfBodyJoints = Body.Rotations.NumOfJoints(); + numOfLeftHandJoints = LeftHand.Rotations.NumOfJoints(); + numOfRightHandJoints = RightHand.Rotations.NumOfJoints(); + } + + public static readonly List metahuman_bones = new List{ + HumanBodyBones.Hips, + HumanBodyBones.RightUpperLeg, + HumanBodyBones.RightLowerLeg, + HumanBodyBones.RightFoot, + HumanBodyBones.RightToes, + HumanBodyBones.LeftUpperLeg, + HumanBodyBones.LeftLowerLeg, + HumanBodyBones.LeftFoot, + HumanBodyBones.LeftToes, + HumanBodyBones.Spine, + HumanBodyBones.Chest, + HumanBodyBones.UpperChest, + HumanBodyBones.LastBone, //Spine4 + HumanBodyBones.LastBone, //Spine5 + HumanBodyBones.Neck, + HumanBodyBones.LastBone, //Neck2 not in unity avatar system, using LastBone as a Null value + HumanBodyBones.Head, + HumanBodyBones.LeftShoulder, + HumanBodyBones.LeftUpperArm, + HumanBodyBones.LeftLowerArm, + HumanBodyBones.RightShoulder, + HumanBodyBones.RightUpperArm, + HumanBodyBones.RightLowerArm, + HumanBodyBones.LeftHand, + HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LeftIndexProximal, + HumanBodyBones.LeftIndexIntermediate, + HumanBodyBones.LeftIndexDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LeftMiddleProximal, + HumanBodyBones.LeftMiddleIntermediate, + HumanBodyBones.LeftMiddleDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LeftRingProximal, + HumanBodyBones.LeftRingIntermediate, + HumanBodyBones.LeftRingDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LeftLittleProximal, + HumanBodyBones.LeftLittleIntermediate, + HumanBodyBones.LeftLittleDistal, + HumanBodyBones.LeftThumbProximal, + HumanBodyBones.LeftThumbIntermediate, + HumanBodyBones.LeftThumbDistal, + HumanBodyBones.RightHand, + HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.RightIndexProximal, + HumanBodyBones.RightIndexIntermediate, + HumanBodyBones.RightIndexDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.RightMiddleProximal, + HumanBodyBones.RightMiddleIntermediate, + HumanBodyBones.RightMiddleDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.RightRingProximal, + HumanBodyBones.RightRingIntermediate, + HumanBodyBones.RightRingDistal, + HumanBodyBones.LastBone, //metacarpal not in unity avatar system, using LastBone as a Null value + HumanBodyBones.RightLittleProximal, + HumanBodyBones.RightLittleIntermediate, + HumanBodyBones.RightLittleDistal, + HumanBodyBones.RightThumbProximal, + HumanBodyBones.RightThumbIntermediate, + HumanBodyBones.RightThumbDistal + }; + public override List GetBones() + { + return metahuman_bones; + } + private static readonly List metaHumanParentIndices = new () { + 0,0,1,2,3,0,5,6,7,0,9,10,11,12,13,14,15,13,17,18,13,20,21,19,19,19,23,26,27,28,23,30,31,32,23,34,35,36,23,38,39,40,23,42,43,22,22,22,45,48,49,50,45,52,53,54,45,56,57,58,45,60,61,62,45,64,65 + }; + + public override Dictionary GetExtraBones() { return new() { + { 12, "spine_04" } , + { 13, "spine_05" } , + { 15, "neck_02" }, + { 24, "lowerarm_twist_01_l"}, + { 25, "lowerarm_twist_02_l" }, + { 26, "index_metacarpal_l" }, + { 30, "middle_metacarpal_l" }, + { 34, "ring_metacarpal_l" }, + { 38, "pinky_metacarpal_l" }, + { 46, "lowerarm_twist_01_r" }, + { 47, "lowerarm_twist_02_r" }, + { 48, "index_metacarpal_r" }, + { 52, "middle_metacarpal_r" }, + { 56, "ring_metacarpal_r" }, + { 60, "pinky_metacarpal_r" }, + + }; } + + public override List GetParentIndices() + { + return metaHumanParentIndices; + } + public override BaseBodyWrapper GetBody() + { + return Body; + } + + public override BaseHandWrapper GetLeftHand() + { + return LeftHand; + } + + public override BaseHandWrapper GetRightHand() + { + return RightHand; + } + } +} + \ No newline at end of file diff --git a/UnityAPI/UnityScripts/PoseAIRigMixamo.cs b/UnityAPI/UnityScripts/PoseAIRigMixamo.cs index ed679e1..78e392d 100644 --- a/UnityAPI/UnityScripts/PoseAIRigMixamo.cs +++ b/UnityAPI/UnityScripts/PoseAIRigMixamo.cs @@ -113,7 +113,17 @@ protected internal override void BuildListsFromIntrospection(){ numOfLeftHandJoints = LeftHand.Rotations.NumOfJoints(); numOfRightHandJoints = RightHand.Rotations.NumOfJoints(); } - + + public override Dictionary GetExtraBones() + { + return new() + { + { 21, "LeftForeArmTwist" }, + { 38, "RightForeArmTwist" }, + }; + } + + public override BaseBodyWrapper GetBody() { return Body; diff --git a/UnityAPI/UnityScripts/PoseAIRigRetarget.cs b/UnityAPI/UnityScripts/PoseAIRigRetarget.cs index 727c568..8543efe 100644 --- a/UnityAPI/UnityScripts/PoseAIRigRetarget.cs +++ b/UnityAPI/UnityScripts/PoseAIRigRetarget.cs @@ -16,8 +16,8 @@ namespace PoseAI [RequireComponent(typeof(Animator))] public class PoseAIRigRetarget : MonoBehaviour - { - + { + [Tooltip("Character animator from a standard rig.")] public Animator OtherAnimator; @@ -38,19 +38,24 @@ private void LateUpdate(){ String mapName = OtherAnimator.name + "_to_" + animator.name; String pasteText = " {\"" + mapName + "\", new List {"; - foreach (var bone in PoseAIRigBase.bones) + foreach (var bone in (new PoseAIRigUnity()).GetBones()) { Quaternion relativeRotation; if (bone == HumanBodyBones.LastBone) { relativeRotation = Quaternion.identity; } - else + else if (animator.GetBoneTransform(bone) != null && OtherAnimator.GetBoneTransform(bone) != null) { var myRotation = animator.GetBoneTransform(bone).rotation; var otherRotation = OtherAnimator.GetBoneTransform(bone).rotation; relativeRotation = Quaternion.Inverse(otherRotation) * myRotation; + } else + { + Debug.LogWarning("Missing " + bone.ToString()); + relativeRotation = rotations[rotations.Count-1]; } + rotations.Add(relativeRotation); pasteText += "new Quaternion(" + relativeRotation.x + "f," + relativeRotation.y + "f," + relativeRotation.z + "f," + relativeRotation.w + "f),"; } diff --git a/UnityAPI/UnityScripts/PoseAIRigUE4.cs b/UnityAPI/UnityScripts/PoseAIRigUE4.cs index 8877c6a..241a6a4 100644 --- a/UnityAPI/UnityScripts/PoseAIRigUE4.cs +++ b/UnityAPI/UnityScripts/PoseAIRigUE4.cs @@ -112,9 +112,16 @@ protected internal override void BuildListsFromIntrospection(){ numOfBodyJoints = Body.Rotations.NumOfJoints(); numOfLeftHandJoints = LeftHand.Rotations.NumOfJoints(); numOfRightHandJoints = RightHand.Rotations.NumOfJoints(); - } - - public override BaseBodyWrapper GetBody() + } + public override Dictionary GetExtraBones() + { + return new() + { + { 21, "lowerarm_twist_01_l" }, + { 38, "lowerarm_twist_01_r" }, + }; + } + public override BaseBodyWrapper GetBody() { return Body; } diff --git a/UnityAPI/UnityScripts/PoseAISourceNative.cs b/UnityAPI/UnityScripts/PoseAISourceNative.cs deleted file mode 100644 index d2b5e0d..0000000 --- a/UnityAPI/UnityScripts/PoseAISourceNative.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2022 Pose AI Ltd. All rights reserved - -using System; -using UnityEngine; - -namespace PoseAI -{ - /* - * A character component which can be used to supply a source to the - * PoseAICharacterController and PoseAIAnimator. This source uses a direct call to - * ProcessNativePacket to provide the PoseAI Engine information as an formated packet. - * This is useful when the Unity game and the engine are on the same device, and could - * be connected with simple messaging (similar to the Unity iOS demo app) - */ - public class PoseAISourceNative : PoseAISource - { - [Tooltip("Format of the rig for streaming. Determines joint names and reference neutral rotations")] - public PoseAI_Rigs RigType = PoseAI_Rigs.Unity; - - private PoseAIRigBase _rigObj; - - public override PoseAIRigBase GetRig() - { - if (_rigObj == null) - _rigObj = PoseAIRigFactory.SelectRig(RigType); - return _rigObj; - } - - public void ProcessNativePacket(string packetAsJsonString) - { - if (!GetRig().OverwriteFromJSON(packetAsJsonString)) - { - Debug.Log("Could not read native packet: " + packetAsJsonString); - } - } - } - -} diff --git a/UnityAPI/UnityScripts/PoseAISourceDirect.cs b/UnityAPI/UnityScripts/PoseAISourceNetwork.cs similarity index 90% rename from UnityAPI/UnityScripts/PoseAISourceDirect.cs rename to UnityAPI/UnityScripts/PoseAISourceNetwork.cs index 3919186..a4cd70e 100644 --- a/UnityAPI/UnityScripts/PoseAISourceDirect.cs +++ b/UnityAPI/UnityScripts/PoseAISourceNetwork.cs @@ -14,17 +14,10 @@ namespace PoseAI { - public abstract class PoseAISource : MonoBehaviour - { - virtual public PoseAIRigBase GetRig() - { - return null; - } - } /* * A component which handles UDP communication with the Pose Camera app. */ - public class PoseAISourceDirect : PoseAISource + public class PoseAISourceNetwork : PoseAISource { [Tooltip("Specify listening port. Must be unique per source and valid for aa port (i.e. 1024-65535)")] public int Port = 8080; @@ -37,12 +30,15 @@ public class PoseAISourceDirect : PoseAISource [Tooltip("Whether to flip left/right and facing like in a mirror. Set false for follow third-person mode.")] public bool MirrorCamera = false; - + + [Tooltip("Whether to predict blendshapes for the face.")] + public bool FaceOn = true; + [Tooltip("Requested app camera framerate. iOS will only use 30 or 60 currently.")] public int CameraFPS = 60; [Tooltip("Requested app streaming interpolation framerate, to smooth animations. Suggest matching target framerate of Unity application.")] - public int SyncFPS = 60; + public int SyncFPS = 0; // each source must have a unique port @@ -85,11 +81,11 @@ private static UdpClient PoseAIUDPClient(int portIn){ private void Start() { if (_usedPorts.Contains(Port)){ - Debug.Log("A PoseAI Source is already using port " + Port.ToString()); + Debug.LogError("A PoseAI Source is already using port " + Port.ToString()); return; } if (Port < 1024 && Port > 65535){ - Debug.Log("A PoseAI Source is set to use an invalid port number:" + Port.ToString()); + Debug.LogError("A PoseAI Source is set to use an invalid port number:" + Port.ToString()); return; } _usedPorts.Add(Port); @@ -136,14 +132,14 @@ private void RunServer() } catch (SocketException ex) { - Debug.Log("PoseAI: SocketException thrown from udp client. " + ex.Message); + Debug.LogWarning("PoseAI: SocketException thrown from udp client. " + ex.Message); if (_currentEndPoint != null && endPoint.Address.Equals(_currentEndPoint.Address)) _currentEndPoint = null; continue; } catch (ObjectDisposedException) { - Debug.Log("PoseAI: ObjectDisposedException thrown from udp client. Restarting"); + Debug.LogWarning("PoseAI: ObjectDisposedException thrown from udp client. Restarting"); if (_currentEndPoint != null && endPoint.Address.Equals(_currentEndPoint.Address)) _currentEndPoint = null; continue; @@ -181,10 +177,11 @@ private void RunServer() private void AcceptPacket(byte[] packet){ if (packet != null){ string packetAsString = Encoding.UTF8.GetString(packet); - + if (!GetRig().OverwriteFromJSON(packetAsString)) { - Debug.Log("Could not read packet from " + _currentEndPoint.ToString()); + Debug.LogWarning("Could not read packet from " + _currentEndPoint.ToString()); + Debug.LogWarning(packetAsString); _currentEndPoint = null; } if (_rigObj.IsIncomingHandshake()) @@ -194,7 +191,7 @@ private void AcceptPacket(byte[] packet){ private void ExchangeHandshake() { - PoseAIHandshake handshakeObj = PoseAIHandshake.Factory(Mode, RigType, MirrorCamera, SyncFPS, CameraFPS); + PoseAIHandshake handshakeObj = PoseAIHandshake.Factory(Mode, RigType, MirrorCamera, FaceOn, SyncFPS, CameraFPS); Debug.Log("Exchanging Handshake"); string handshake = JsonUtility.ToJson(handshakeObj, true); Debug.Log(handshake); diff --git a/UnityAPI/UnityScripts/PoseAISourceShared.cs b/UnityAPI/UnityScripts/PoseAISourceShared.cs index d0c47bf..af74949 100644 --- a/UnityAPI/UnityScripts/PoseAISourceShared.cs +++ b/UnityAPI/UnityScripts/PoseAISourceShared.cs @@ -10,10 +10,17 @@ namespace PoseAI * PoseAICharacterController and PoseAIAnimator. Create a direct source elsewhere * and then drag into this component. */ + public abstract class PoseAISource : MonoBehaviour + { + virtual public PoseAIRigBase GetRig() + { + return null; + } + } public class PoseAISourceShared : PoseAISource { [Tooltip("Specify a direct source created elsewhere.")] - public PoseAISourceDirect Source; + public PoseAISource Source; public override PoseAIRigBase GetRig() { return Source.GetRig(); diff --git a/UnityAPI/UnityScripts/PoseAIUtility.cs b/UnityAPI/UnityScripts/PoseAIUtility.cs index 178f978..6173a4f 100644 --- a/UnityAPI/UnityScripts/PoseAIUtility.cs +++ b/UnityAPI/UnityScripts/PoseAIUtility.cs @@ -27,7 +27,7 @@ public class PoseAIRuntimeLoader * 3. either a new port for a new source, or an exsiting PoseAISource (i.e. if player is already connected and we are animating a new character or scene). * 4. optionally pass in a different animation controller if you want to use different animation controllers in different parts of the game */ - public static bool RuntimeLoad(GameObject gameCharacter, Avatar avatar, int port = 0, PoseAISourceDirect poseAISourceIn = null, RuntimeAnimatorController animationController = null) + public static bool RuntimeLoad(GameObject gameCharacter, Avatar avatar, int port = 0, PoseAISourceNetwork poseAISourceIn = null, RuntimeAnimatorController animationController = null) { Debug.Assert(poseAISourceIn != null || port != 0); @@ -76,9 +76,9 @@ public static bool RuntimeLoad(GameObject gameCharacter, Avatar avatar, int port } else { - PoseAISourceDirect poseAISourceDirect = gameCharacter.GetComponent(); + PoseAISourceNetwork poseAISourceDirect = gameCharacter.GetComponent(); if (poseAISourceDirect == null) - poseAISourceDirect = gameCharacter.AddComponent() as PoseAISourceDirect; + poseAISourceDirect = gameCharacter.AddComponent() as PoseAISourceNetwork; poseAISourceDirect.Mode = PoseAI_Modes.Room; poseAISourceDirect.RigType = rigType; poseAISourceDirect.Port = port; @@ -94,8 +94,7 @@ public static bool RuntimeLoad(GameObject gameCharacter, Avatar avatar, int port poseAICharacterAnimator = gameCharacter.AddComponent() as PoseAICharacterAnimator; poseAICharacterAnimator.SetSource(poseAISource); poseAICharacterAnimator.SetRemapping(Remapping); - poseAICharacterAnimator.OverrideRootName = ""; - poseAICharacterAnimator.JointNamePrefix = ""; + poseAICharacterAnimator.OverridePelvisName = ""; PoseAICharacterController poseAICharacterController = gameCharacter.GetComponent(); if (poseAICharacterController == null) @@ -139,7 +138,7 @@ public static bool RuntimeLoad(GameObject gameCharacter, Avatar avatar, int port public void EnableOrDisablePoseAI(GameObject gameCharacter, bool enabled) { - PoseAISourceDirect poseAISource = gameCharacter.GetComponent(); + PoseAISourceNetwork poseAISource = gameCharacter.GetComponent(); if (poseAISource != null) poseAISource.enabled = enabled; @@ -183,13 +182,27 @@ public static float FixedB64pairToFloat(char a, char b) return firstByte[CharToInt(a)] + secondByte[CharToInt(b)]; } - public static void FStringFixed12ToFloat(ref string data, ref List flatArray) + public static void FStringFixed12ToFloatInPlace(ref string data, ref List flatArray) { + for (int i = 0; i < data.Length - 1; i += 2) + flatArray[i/2] = FixedB64pairToFloat(data[i], data[i + 1]); + } + public static void FStringFixed12ToFloat(ref string data, ref List flatArray, bool clearFirst=false) + { + if (clearFirst) + flatArray.Clear(); flatArray.Capacity = flatArray.Count + data.Length / 2; for (int i = 0; i < data.Length - 1; i += 2) flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); } - + public static void FStringFixed12ToFloat(string data, ref List flatArray, float scale=1.0f, bool clearFirst = false) + { + if (clearFirst) + flatArray.Clear(); + flatArray.Capacity = flatArray.Count + data.Length / 2; + for (int i = 0; i < data.Length - 1; i += 2) + flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1]) * scale); + } public static void FStringFixed12ToFloat(ref string data, ref List flatArray) { flatArray.Capacity = flatArray.Count + data.Length / 2; @@ -215,6 +228,15 @@ public static void FlatArrayToQuats(ref List flatArray, ref List PoseAILiveLinkSingleSource::usedPorts = {}; - - -PoseAILiveLinkSingleSource::PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake) : - port(port), status(LOCTEXT("statusConnecting", "connecting")) -{ - dispatcher = UPoseAIEventDispatcher::GetDispatcher(); - - UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); - udpServer = MakeShared(handshake, this, useIPv6, port); - if (udpServer.IsValid()) { - dispatcher->handshakeUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SetHandshake); - dispatcher->modelConfigUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SendConfig); - dispatcher->disconnect.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::DisconnectTarget); - dispatcher->closeSource.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::CloseTarget); - if (useIPv6) { - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); - } - else { - FString myIP; - udpServer->GetIP(myIP); - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); - } - } - else { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: unable to create a server on %d using IPv4"), port); - } -} -void PoseAILiveLinkSingleSource::Update() { - if (udpServer->hasNewRig) { - AddSubject(udpServer->rig); - udpServer->hasNewRig = false; - } -} - - -bool PoseAILiveLinkSingleSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { - - bool bResult = false; - if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) - { - FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); - - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); - FGuid existingSource; - if (PoseAILiveLinkSingleSource::GetPortGuid(portNum, existingSource)) - LiveLinkClient.RemoveSource(existingSource); - } - TSharedPtr PoseAISource = MakeShared(portNum, isIPv6, handshake); - TSharedPtr Source = StaticCastSharedPtr(PoseAISource); - LiveLinkClient.AddSource(Source); - LiveLinkClient.Tick(); - subjectName = PoseAISource->GetSubjectName(); - bResult = true; - } - return bResult; -} - -void PoseAILiveLinkSingleSource::AddSubject(TSharedPtr rig){ - - check(IsInGameThread()); - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - FLiveLinkSubjectPreset subject; - subject.bEnabled = true; - subject.Key = subjectKey; - subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); - subject.Settings = nullptr; - subject.VirtualSubject = nullptr; - - critSingleSection.Lock(); - if (!liveLinkClient->CreateSubject(subject)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - else { - UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); - FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); - liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); - - } - critSingleSection.Unlock(); -} - -bool PoseAILiveLinkSingleSource::GetPortGuid(int32 port, FGuid& fguid) { - bool has = usedPorts.Contains(port); - if (has) - fguid = usedPorts[port].source; - return has; -} - -FName PoseAILiveLinkSingleSource::SubjectNameFromPort(int32 port) { - FString NewString = FString("PoseCam@port:") + FString::FromInt(port); - return FName(*NewString); - -} - -void PoseAILiveLinkSingleSource::SetConnectionName(FName name) { - if (usedPorts.Contains(port)) - usedPorts[port].connectionName = name; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(int32 port) { - return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(const FLiveLinkSubjectName& name) { - for (const auto& elem : usedPorts) { - if (elem.Value.subjectKey.SubjectName == name) - return elem.Value.connectionName; - } - return NAME_None; -} - -bool PoseAILiveLinkSingleSource::IsSourceStillValid() const { return true; } - -bool PoseAILiveLinkSingleSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } - -void PoseAILiveLinkSingleSource::disable() -{ - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: disabling the source")); - status = LOCTEXT("statusDisabled", "disabled"); - liveLinkClient = nullptr; -} - - -void PoseAILiveLinkSingleSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) -{ - sourceGuid = InSourceGuid; - subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); - PoseAIPortRecord record = PoseAIPortRecord(); - record.source = InSourceGuid; - record.subjectKey = subjectKey; - usedPorts.Add(port, record); - client = InClient; - liveLinkClient = InClient; - - if (udpServer->rig != nullptr && udpServer->rig.IsValid()) - AddSubject(udpServer->rig); - - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: receive client %s"), *client->ModularFeatureName.ToString()); -} - -bool PoseAILiveLinkSingleSource::RequestSourceShutdown() -{ - usedPorts.Remove(port); - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkSingleSource on port %d closed"), port); - if (liveLinkClient != nullptr) { - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - liveLinkClient->RemoveSource(sourceGuid); - liveLinkClient = nullptr; - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - return true; -} - -void PoseAILiveLinkSingleSource::UpdatePose(TSharedPtr rig, TSharedPtr jsonPose) -{ - if (liveLinkClient == nullptr) - return; - if(rig == nullptr || !rig.IsValid()) { - return; - } - FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); - FLiveLinkAnimationFrameData& data = *frameData.Cast(); - data.Transforms.Reserve(100); - if (rig->ProcessFrame(jsonPose, data)) { - liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); - } -} diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Config/FilterPlugin.ini b/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Config/FilterPlugin.ini deleted file mode 100644 index ccebca2..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Config/FilterPlugin.ini +++ /dev/null @@ -1,8 +0,0 @@ -[FilterPlugin] -; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and -; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively. -; -; Examples: -; /README.txt -; /Extras/... -; /Binaries/ThirdParty/*.dll diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp deleted file mode 100644 index d7eb764..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLink.h" - -//#define LOCTEXT_NAMESPACE "FPoseAILiveLinkModule" - -void FPoseAILiveLinkModule::StartupModule() -{ - // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module -} - -void FPoseAILiveLinkModule::ShutdownModule() -{ - // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, - // we call this function before unloading the module. -} - -//#undef LOCTEXT_NAMESPACE - -IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp b/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp deleted file mode 100644 index 48e8e93..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLinkSingleSource.h" -#include "Features/IModularFeatures.h" - - - FCriticalSection critSingleSection; - - TMap PoseAILiveLinkSingleSource::usedPorts = {}; - - -PoseAILiveLinkSingleSource::PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake) : - port(port), status(LOCTEXT("statusConnecting", "connecting")) -{ - dispatcher = UPoseAIEventDispatcher::GetDispatcher(); - - UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); - udpServer = MakeShared(handshake, this, useIPv6, port); - if (udpServer.IsValid()) { - dispatcher->handshakeUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SetHandshake); - dispatcher->modelConfigUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SendConfig); - dispatcher->disconnect.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::DisconnectTarget); - dispatcher->closeSource.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::CloseTarget); - if (useIPv6) { - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); - } - else { - FString myIP; - udpServer->GetIP(myIP); - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); - } - } - else { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: unable to create a server on %d using IPv4"), port); - } -} -void PoseAILiveLinkSingleSource::Update() { - if (udpServer->hasNewRig) { - AddSubject(udpServer->rig); - udpServer->hasNewRig = false; - } -} - - -bool PoseAILiveLinkSingleSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { - - bool bResult = false; - if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) - { - FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); - - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); - FGuid existingSource; - if (PoseAILiveLinkSingleSource::GetPortGuid(portNum, existingSource)) - LiveLinkClient.RemoveSource(existingSource); - } - TSharedPtr PoseAISource = MakeShared(portNum, isIPv6, handshake); - TSharedPtr Source = StaticCastSharedPtr(PoseAISource); - LiveLinkClient.AddSource(Source); - LiveLinkClient.Tick(); - subjectName = PoseAISource->GetSubjectName(); - bResult = true; - } - return bResult; -} - -void PoseAILiveLinkSingleSource::AddSubject(TSharedPtr rig){ - - check(IsInGameThread()); - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - FLiveLinkSubjectPreset subject; - subject.bEnabled = true; - subject.Key = subjectKey; - subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); - subject.Settings = nullptr; - subject.VirtualSubject = nullptr; - - critSingleSection.Lock(); - if (!liveLinkClient->CreateSubject(subject)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - else { - UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); - FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); - liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); - - } - critSingleSection.Unlock(); -} - -bool PoseAILiveLinkSingleSource::GetPortGuid(int32 port, FGuid& fguid) { - bool has = usedPorts.Contains(port); - if (has) - fguid = usedPorts[port].source; - return has; -} - -FName PoseAILiveLinkSingleSource::SubjectNameFromPort(int32 port) { - FString NewString = FString("PoseCam@port:") + FString::FromInt(port); - return FName(*NewString); - -} - -void PoseAILiveLinkSingleSource::SetConnectionName(FName name) { - if (usedPorts.Contains(port)) - usedPorts[port].connectionName = name; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(int32 port) { - return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(const FLiveLinkSubjectName& name) { - for (const auto& elem : usedPorts) { - if (elem.Value.subjectKey.SubjectName == name) - return elem.Value.connectionName; - } - return NAME_None; -} - -bool PoseAILiveLinkSingleSource::IsSourceStillValid() const { return true; } - -bool PoseAILiveLinkSingleSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } - -void PoseAILiveLinkSingleSource::disable() -{ - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: disabling the source")); - status = LOCTEXT("statusDisabled", "disabled"); - liveLinkClient = nullptr; -} - - -void PoseAILiveLinkSingleSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) -{ - sourceGuid = InSourceGuid; - subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); - PoseAIPortRecord record = PoseAIPortRecord(); - record.source = InSourceGuid; - record.subjectKey = subjectKey; - usedPorts.Add(port, record); - client = InClient; - liveLinkClient = InClient; - - if (udpServer->rig != nullptr && udpServer->rig.IsValid()) - AddSubject(udpServer->rig); - - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: receive client %s"), *client->ModularFeatureName.ToString()); -} - -bool PoseAILiveLinkSingleSource::RequestSourceShutdown() -{ - usedPorts.Remove(port); - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkSingleSource on port %d closed"), port); - if (liveLinkClient != nullptr) { - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - liveLinkClient->RemoveSource(sourceGuid); - liveLinkClient = nullptr; - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - return true; -} - -void PoseAILiveLinkSingleSource::UpdatePose(TSharedPtr rig, TSharedPtr jsonPose) -{ - if (liveLinkClient == nullptr) - return; - if(rig == nullptr || !rig.IsValid()) { - return; - } - FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); - FLiveLinkAnimationFrameData& data = *frameData.Cast(); - data.Transforms.Reserve(100); - if (rig->ProcessFrame(jsonPose, data)) { - liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); - } -} diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Config/FilterPlugin.ini b/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Config/FilterPlugin.ini deleted file mode 100644 index ccebca2..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Config/FilterPlugin.ini +++ /dev/null @@ -1,8 +0,0 @@ -[FilterPlugin] -; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and -; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively. -; -; Examples: -; /README.txt -; /Extras/... -; /Binaries/ThirdParty/*.dll diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp deleted file mode 100644 index d7eb764..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLink.h" - -//#define LOCTEXT_NAMESPACE "FPoseAILiveLinkModule" - -void FPoseAILiveLinkModule::StartupModule() -{ - // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module -} - -void FPoseAILiveLinkModule::ShutdownModule() -{ - // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, - // we call this function before unloading the module. -} - -//#undef LOCTEXT_NAMESPACE - -IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp b/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp deleted file mode 100644 index 48e8e93..0000000 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLinkSingleSource.h" -#include "Features/IModularFeatures.h" - - - FCriticalSection critSingleSection; - - TMap PoseAILiveLinkSingleSource::usedPorts = {}; - - -PoseAILiveLinkSingleSource::PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake) : - port(port), status(LOCTEXT("statusConnecting", "connecting")) -{ - dispatcher = UPoseAIEventDispatcher::GetDispatcher(); - - UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); - udpServer = MakeShared(handshake, this, useIPv6, port); - if (udpServer.IsValid()) { - dispatcher->handshakeUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SetHandshake); - dispatcher->modelConfigUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SendConfig); - dispatcher->disconnect.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::DisconnectTarget); - dispatcher->closeSource.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::CloseTarget); - if (useIPv6) { - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); - } - else { - FString myIP; - udpServer->GetIP(myIP); - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); - } - } - else { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: unable to create a server on %d using IPv4"), port); - } -} -void PoseAILiveLinkSingleSource::Update() { - if (udpServer->hasNewRig) { - AddSubject(udpServer->rig); - udpServer->hasNewRig = false; - } -} - - -bool PoseAILiveLinkSingleSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { - - bool bResult = false; - if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) - { - FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); - - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); - FGuid existingSource; - if (PoseAILiveLinkSingleSource::GetPortGuid(portNum, existingSource)) - LiveLinkClient.RemoveSource(existingSource); - } - TSharedPtr PoseAISource = MakeShared(portNum, isIPv6, handshake); - TSharedPtr Source = StaticCastSharedPtr(PoseAISource); - LiveLinkClient.AddSource(Source); - LiveLinkClient.Tick(); - subjectName = PoseAISource->GetSubjectName(); - bResult = true; - } - return bResult; -} - -void PoseAILiveLinkSingleSource::AddSubject(TSharedPtr rig){ - - check(IsInGameThread()); - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - FLiveLinkSubjectPreset subject; - subject.bEnabled = true; - subject.Key = subjectKey; - subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); - subject.Settings = nullptr; - subject.VirtualSubject = nullptr; - - critSingleSection.Lock(); - if (!liveLinkClient->CreateSubject(subject)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - else { - UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); - FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); - liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); - - } - critSingleSection.Unlock(); -} - -bool PoseAILiveLinkSingleSource::GetPortGuid(int32 port, FGuid& fguid) { - bool has = usedPorts.Contains(port); - if (has) - fguid = usedPorts[port].source; - return has; -} - -FName PoseAILiveLinkSingleSource::SubjectNameFromPort(int32 port) { - FString NewString = FString("PoseCam@port:") + FString::FromInt(port); - return FName(*NewString); - -} - -void PoseAILiveLinkSingleSource::SetConnectionName(FName name) { - if (usedPorts.Contains(port)) - usedPorts[port].connectionName = name; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(int32 port) { - return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(const FLiveLinkSubjectName& name) { - for (const auto& elem : usedPorts) { - if (elem.Value.subjectKey.SubjectName == name) - return elem.Value.connectionName; - } - return NAME_None; -} - -bool PoseAILiveLinkSingleSource::IsSourceStillValid() const { return true; } - -bool PoseAILiveLinkSingleSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } - -void PoseAILiveLinkSingleSource::disable() -{ - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: disabling the source")); - status = LOCTEXT("statusDisabled", "disabled"); - liveLinkClient = nullptr; -} - - -void PoseAILiveLinkSingleSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) -{ - sourceGuid = InSourceGuid; - subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); - PoseAIPortRecord record = PoseAIPortRecord(); - record.source = InSourceGuid; - record.subjectKey = subjectKey; - usedPorts.Add(port, record); - client = InClient; - liveLinkClient = InClient; - - if (udpServer->rig != nullptr && udpServer->rig.IsValid()) - AddSubject(udpServer->rig); - - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: receive client %s"), *client->ModularFeatureName.ToString()); -} - -bool PoseAILiveLinkSingleSource::RequestSourceShutdown() -{ - usedPorts.Remove(port); - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkSingleSource on port %d closed"), port); - if (liveLinkClient != nullptr) { - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - liveLinkClient->RemoveSource(sourceGuid); - liveLinkClient = nullptr; - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - return true; -} - -void PoseAILiveLinkSingleSource::UpdatePose(TSharedPtr rig, TSharedPtr jsonPose) -{ - if (liveLinkClient == nullptr) - return; - if(rig == nullptr || !rig.IsValid()) { - return; - } - FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); - FLiveLinkAnimationFrameData& data = *frameData.Cast(); - data.Transforms.Reserve(100); - if (rig->ProcessFrame(jsonPose, data)) { - liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); - } -} diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Config/FilterPlugin.ini b/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Config/FilterPlugin.ini deleted file mode 100644 index ccebca2..0000000 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Config/FilterPlugin.ini +++ /dev/null @@ -1,8 +0,0 @@ -[FilterPlugin] -; This section lists additional files which will be packaged along with your plugin. Paths should be listed relative to the root plugin directory, and -; may include "...", "*", and "?" wildcards to match directories, files, and individual characters respectively. -; -; Examples: -; /README.txt -; /Extras/... -; /Binaries/ThirdParty/*.dll diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp deleted file mode 100644 index d7eb764..0000000 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLink.h" - -//#define LOCTEXT_NAMESPACE "FPoseAILiveLinkModule" - -void FPoseAILiveLinkModule::StartupModule() -{ - // This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module -} - -void FPoseAILiveLinkModule::ShutdownModule() -{ - // This function may be called during shutdown to clean up your module. For modules that support dynamic reloading, - // we call this function before unloading the module. -} - -//#undef LOCTEXT_NAMESPACE - -IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp b/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp deleted file mode 100644 index 48e8e93..0000000 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSingleSource.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright Pose AI Ltd 2021 - -#include "PoseAILiveLinkSingleSource.h" -#include "Features/IModularFeatures.h" - - - FCriticalSection critSingleSection; - - TMap PoseAILiveLinkSingleSource::usedPorts = {}; - - -PoseAILiveLinkSingleSource::PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake) : - port(port), status(LOCTEXT("statusConnecting", "connecting")) -{ - dispatcher = UPoseAIEventDispatcher::GetDispatcher(); - - UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); - udpServer = MakeShared(handshake, this, useIPv6, port); - if (udpServer.IsValid()) { - dispatcher->handshakeUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SetHandshake); - dispatcher->modelConfigUpdate.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::SendConfig); - dispatcher->disconnect.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::DisconnectTarget); - dispatcher->closeSource.AddSP(udpServer.ToSharedRef(), &PoseAILiveLinkServer::CloseTarget); - if (useIPv6) { - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); - } - else { - FString myIP; - udpServer->GetIP(myIP); - status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); - } - } - else { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: unable to create a server on %d using IPv4"), port); - } -} -void PoseAILiveLinkSingleSource::Update() { - if (udpServer->hasNewRig) { - AddSubject(udpServer->rig); - udpServer->hasNewRig = false; - } -} - - -bool PoseAILiveLinkSingleSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { - - bool bResult = false; - if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) - { - FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); - - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); - FGuid existingSource; - if (PoseAILiveLinkSingleSource::GetPortGuid(portNum, existingSource)) - LiveLinkClient.RemoveSource(existingSource); - } - TSharedPtr PoseAISource = MakeShared(portNum, isIPv6, handshake); - TSharedPtr Source = StaticCastSharedPtr(PoseAISource); - LiveLinkClient.AddSource(Source); - LiveLinkClient.Tick(); - subjectName = PoseAISource->GetSubjectName(); - bResult = true; - } - return bResult; -} - -void PoseAILiveLinkSingleSource::AddSubject(TSharedPtr rig){ - - check(IsInGameThread()); - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - FLiveLinkSubjectPreset subject; - subject.bEnabled = true; - subject.Key = subjectKey; - subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); - subject.Settings = nullptr; - subject.VirtualSubject = nullptr; - - critSingleSection.Lock(); - if (!liveLinkClient->CreateSubject(subject)) { - UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - else { - UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); - FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); - liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); - - } - critSingleSection.Unlock(); -} - -bool PoseAILiveLinkSingleSource::GetPortGuid(int32 port, FGuid& fguid) { - bool has = usedPorts.Contains(port); - if (has) - fguid = usedPorts[port].source; - return has; -} - -FName PoseAILiveLinkSingleSource::SubjectNameFromPort(int32 port) { - FString NewString = FString("PoseCam@port:") + FString::FromInt(port); - return FName(*NewString); - -} - -void PoseAILiveLinkSingleSource::SetConnectionName(FName name) { - if (usedPorts.Contains(port)) - usedPorts[port].connectionName = name; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(int32 port) { - return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; -} - -FName PoseAILiveLinkSingleSource::GetConnectionName(const FLiveLinkSubjectName& name) { - for (const auto& elem : usedPorts) { - if (elem.Value.subjectKey.SubjectName == name) - return elem.Value.connectionName; - } - return NAME_None; -} - -bool PoseAILiveLinkSingleSource::IsSourceStillValid() const { return true; } - -bool PoseAILiveLinkSingleSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } - -void PoseAILiveLinkSingleSource::disable() -{ - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: disabling the source")); - status = LOCTEXT("statusDisabled", "disabled"); - liveLinkClient = nullptr; -} - - -void PoseAILiveLinkSingleSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) -{ - sourceGuid = InSourceGuid; - subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); - PoseAIPortRecord record = PoseAIPortRecord(); - record.source = InSourceGuid; - record.subjectKey = subjectKey; - usedPorts.Add(port, record); - client = InClient; - liveLinkClient = InClient; - - if (udpServer->rig != nullptr && udpServer->rig.IsValid()) - AddSubject(udpServer->rig); - - UE_LOG(LogTemp, Display, TEXT("Pose AI LiveLink: receive client %s"), *client->ModularFeatureName.ToString()); -} - -bool PoseAILiveLinkSingleSource::RequestSourceShutdown() -{ - usedPorts.Remove(port); - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkSingleSource on port %d closed"), port); - if (liveLinkClient != nullptr) { - liveLinkClient->RemoveSubject_AnyThread(subjectKey); - liveLinkClient->RemoveSource(sourceGuid); - liveLinkClient = nullptr; - UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); - } - return true; -} - -void PoseAILiveLinkSingleSource::UpdatePose(TSharedPtr rig, TSharedPtr jsonPose) -{ - if (liveLinkClient == nullptr) - return; - if(rig == nullptr || !rig.IsValid()) { - return; - } - FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); - FLiveLinkAnimationFrameData& data = *frameData.Cast(); - data.Transforms.Reserve(100); - if (rig->ProcessFrame(jsonPose, data)) { - liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); - } -} diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/PoseAILiveLink.uplugin index f85260a..3430354 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,7 +1,7 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.41", + "VersionName": "1.42", "FriendlyName": "PoseAI LiveLink", "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 6b53b2d..61106be 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -93,6 +93,9 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -143,25 +146,29 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { -bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { portNum = PoseAILiveLinkNetworkSource::portDefault; while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; } - return AddSource(handshake, myIP, portNum, isIPv6); + return AddSource(handshake, isIPv6, portNum, myIP, subject); } -bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { - FLiveLinkSubjectName addedSubjectName; +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); } + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -210,7 +217,9 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..f68e3da --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + + if (jsonPose != nullptr && jsonPose->HasTypedField("Face")) { + + + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp index 58ac335..61c68f1 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -6,9 +6,6 @@ -FCriticalSection critSingleSectionLocal; - - /* First use the static method to create a source and add it to the LiveLinkClient. * The LiveLinkClient must own only shared pointer or UE will crash on cleanup. * The client will respond with receive client when it is registered. We return a weak ptr @@ -51,6 +48,9 @@ void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid sourceGuid = InSourceGuid; subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); } /* @@ -113,6 +113,7 @@ bool PoseAILiveLinkNativeSource::RequestSourceShutdown() { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; @@ -150,7 +151,8 @@ void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); - + faceSubSource->UpdateFace(jsonPose); } } } + diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp index ff08b49..b18a1a0 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -4,7 +4,6 @@ #include "Features/IModularFeatures.h" #include "PoseAIEventDispatcher.h" -FCriticalSection critSingleSection; /* @@ -89,6 +88,9 @@ void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid liveLinkClient = InClient; AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + } /* @@ -119,7 +121,7 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ subject.Settings = nullptr; subject.VirtualSubject = nullptr; - critSingleSection.Lock(); + FScopeLock ScopeLock(&InSynchObject); if (!liveLinkClient->CreateSubject(subject)) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); } @@ -128,7 +130,6 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); } - critSingleSection.Unlock(); } @@ -145,6 +146,7 @@ void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) data.Transforms.Reserve(100); if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); } else { static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; @@ -212,6 +214,7 @@ bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() usedPorts.Remove(port); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index fb05e4b..8d2646d 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -24,14 +24,14 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), rigType(FName(handshake.GetRigString())), - useRootMotion(handshake.useRootMotion), + useRootMotion(handshake.useRootMotion), includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; switch (handshake.rig) { @@ -41,6 +41,9 @@ TSharedPtr PoseAIRig::PoseAIRigFactory(const FLi case EPoseAiRigPresets::Mixamo: rigPtr = MakeShared(name, handshake); break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; case EPoseAiRigPresets::DazUE: rigPtr = MakeShared(name, handshake); break; @@ -86,6 +89,7 @@ bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) { + double timestamp; jsonObject->TryGetNumberField("Timestamp", timestamp); // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. @@ -116,7 +120,7 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink liveValues.rootTranslation = FVector( -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip 0.0f, //currently no body distance estimate from pose camera - 0.0f + 0.0f // could do this if game calibrates from a player starting position: liveValues.hipScreen[1] * rigHeight / liveValues.bodyHeight ); TriggerEvents(); @@ -222,21 +226,34 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { + FVector baseTranslation = FVector::ZeroVector; + // seperating into lowest body part in case we want to connect with AI later + float minTorso = 0.0f; + float minFeetZ = 0.0f; + float minKnees = 0.0f; + float minHands = 0.0f; + + if (!isDesktop) baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - TArray componentTransform; - componentTransform.Emplace(data.Transforms[0]); - componentTransform.Emplace(data.Transforms[1]); - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } + + TArray componentTransform; + componentTransform.Emplace(data.Transforms[0]); + componentTransform.Emplace(data.Transforms[1]); + // to ensure grounding in the capsule, calculates lowest Z in component space. Does not include hands + for (int32 j = 2; j < parentIndices.Num(); j++) { + componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); + if (j == rShinJoint + 1 || j == rShinJoint + 2 || j == lShinJoint + 1 || j == lShinJoint + 2) + minFeetZ = FGenericPlatformMath::Min(minFeetZ, componentTransform[j].GetTranslation().Z); + else if (j == rShinJoint || j == lShinJoint) + minKnees = FGenericPlatformMath::Min(minKnees, componentTransform[j].GetTranslation().Z); + else if (j >= numBodyJoints) + minHands = FGenericPlatformMath::Min(minHands, componentTransform[j].GetTranslation().Z); + else + minTorso = FGenericPlatformMath::Min(minTorso, componentTransform[j].GetTranslation().Z); } + + float minZ = FMath::Min(FMath::Min(minTorso, minHands), FMath::Min(minFeetZ, minKnees)); + minZ -= rootHipOffsetZ; // assigns motion either to root or to hips @@ -291,11 +308,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -383,7 +404,7 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject data.Transforms.Add(transform); } - AssignCharacterMotion(data); + AssignCharacterMotion(data); CachePose(data.Transforms); hasProcessedRotations = true; @@ -428,6 +449,15 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -618,6 +648,83 @@ void PoseAIRigMixamo::Configure() rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -711,7 +818,11 @@ void PoseAIRigMetaHuman::Configure() void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 4; + lShinJoint = 9; + lowerBodyNumOfJoints = 10; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index 595a509..28e63ac 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -85,6 +85,10 @@ int32 FPoseAIHandshake::GetHandModelVersion() const { return static_cast(handModelVersion) + 1; } +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + FString FPoseAIHandshake::GetModeString() const { switch (mode) { case EPoseAiAppModes::Room: return "Room"; @@ -101,6 +105,7 @@ FString FPoseAIHandshake::GetRigString() const { case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; case EPoseAiRigPresets::UE4: return "UE4"; case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; case EPoseAiRigPresets::DazUE: return "DazUE"; default: return "MetaHuman"; @@ -117,24 +122,30 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), *(GetRigString()), *(GetModeString()), + *(YesNoString(isFaceAnimating)), *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, + GetBodyModelVersion(), GetHandModelVersion(), + *(YesNoString(locomotionEvents)), static_cast(packetFormat) ); } diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index fdf95f7..7403779 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -141,6 +141,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -268,6 +272,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -303,16 +309,22 @@ GENERATED_BODY() /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum = 8080, bool isIPv6 = false); + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..7029303 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,107 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h index 854c15d..be7e999 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -18,6 +18,7 @@ #include "Json.h" #include "PoseAIRig.h" #include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -72,6 +73,8 @@ class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource ILiveLinkClient* liveLinkClient = nullptr; FCriticalSection InSynchObject; FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + mutable FText status; }; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index b959395..f11d9a4 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -19,7 +19,7 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" - +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -102,9 +102,10 @@ class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - + TUniquePtr faceSubSource; mutable FText status; - + FCriticalSection InSynchObject; + void AddSubject(); }; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 1bfa6ba..5aa5894 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -64,12 +64,16 @@ class POSEAILIVELINK_API PoseAIRig bool useRootMotion; bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; + bool isCrouching = false; int32 handZoneL = 5; @@ -89,7 +93,7 @@ class POSEAILIVELINK_API PoseAIRig float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); @@ -102,6 +106,7 @@ class POSEAILIVELINK_API PoseAIRig void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +123,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; diff --git a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index 27fb37f..2f7ee9e 100644 --- a/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV1.4/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -40,13 +40,18 @@ enum class EPoseAiContext : uint8 UENUM(BlueprintType) enum class EPoseAiRigPresets : uint8 { - MetaHuman, UE4, Mixamo, DazUE + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt }; UENUM(BlueprintType) enum class EPoseAiHandModel : uint8 { Version1, Version2_EXPERIMENTAL }; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; @@ -62,47 +67,60 @@ struct POSEAILIVELINK_API FPoseAIHandshake /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - EPoseAiRigPresets rig = EPoseAiRigPresets::UE4; + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = true; + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; - + bool useRootMotion = false; /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 cameraFPS = 60; - /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 syncFPS = 0; - - /* controls compactness of packet. */ + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; - + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; - /* the model context. Will enable new AI models as they are deployed*/ + /* the model context. Reserved for future AI models*/ UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } @@ -111,6 +129,7 @@ struct POSEAILIVELINK_API FPoseAIHandshake FString GetContextString() const; FString GetModeString() const; FString GetRigString() const; + int32 GetBodyModelVersion() const; int32 GetHandModelVersion() const; FString ToString() const; FString YesNoString(bool val) const { diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/PoseAILiveLink.uplugin index 9fafb76..77fd4d6 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,7 +1,7 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.40", + "VersionName": "1.42", "FriendlyName": "PoseAI LiveLink", "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 6b53b2d..61106be 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -93,6 +93,9 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -143,25 +146,29 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { -bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { portNum = PoseAILiveLinkNetworkSource::portDefault; while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; } - return AddSource(handshake, myIP, portNum, isIPv6); + return AddSource(handshake, isIPv6, portNum, myIP, subject); } -bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { - FLiveLinkSubjectName addedSubjectName; +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); } + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -210,7 +217,9 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..f68e3da --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + + if (jsonPose != nullptr && jsonPose->HasTypedField("Face")) { + + + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp index 58ac335..61c68f1 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -6,9 +6,6 @@ -FCriticalSection critSingleSectionLocal; - - /* First use the static method to create a source and add it to the LiveLinkClient. * The LiveLinkClient must own only shared pointer or UE will crash on cleanup. * The client will respond with receive client when it is registered. We return a weak ptr @@ -51,6 +48,9 @@ void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid sourceGuid = InSourceGuid; subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); } /* @@ -113,6 +113,7 @@ bool PoseAILiveLinkNativeSource::RequestSourceShutdown() { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; @@ -150,7 +151,8 @@ void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); - + faceSubSource->UpdateFace(jsonPose); } } } + diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp index ff08b49..b18a1a0 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -4,7 +4,6 @@ #include "Features/IModularFeatures.h" #include "PoseAIEventDispatcher.h" -FCriticalSection critSingleSection; /* @@ -89,6 +88,9 @@ void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid liveLinkClient = InClient; AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + } /* @@ -119,7 +121,7 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ subject.Settings = nullptr; subject.VirtualSubject = nullptr; - critSingleSection.Lock(); + FScopeLock ScopeLock(&InSynchObject); if (!liveLinkClient->CreateSubject(subject)) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); } @@ -128,7 +130,6 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); } - critSingleSection.Unlock(); } @@ -145,6 +146,7 @@ void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) data.Transforms.Reserve(100); if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); } else { static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; @@ -212,6 +214,7 @@ bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() usedPorts.Remove(port); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 5989de6..8d2646d 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -24,14 +24,14 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), rigType(FName(handshake.GetRigString())), - useRootMotion(handshake.useRootMotion), + useRootMotion(handshake.useRootMotion), includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; switch (handshake.rig) { @@ -41,6 +41,9 @@ TSharedPtr PoseAIRig::PoseAIRigFactory(const FLi case EPoseAiRigPresets::Mixamo: rigPtr = MakeShared(name, handshake); break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; case EPoseAiRigPresets::DazUE: rigPtr = MakeShared(name, handshake); break; @@ -117,7 +120,7 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink liveValues.rootTranslation = FVector( -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip 0.0f, //currently no body distance estimate from pose camera - 0.0f + 0.0f // could do this if game calibrates from a player starting position: liveValues.hipScreen[1] * rigHeight / liveValues.bodyHeight ); TriggerEvents(); @@ -223,21 +226,34 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { + FVector baseTranslation = FVector::ZeroVector; + // seperating into lowest body part in case we want to connect with AI later + float minTorso = 0.0f; + float minFeetZ = 0.0f; + float minKnees = 0.0f; + float minHands = 0.0f; + + if (!isDesktop) baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - TArray componentTransform; - componentTransform.Emplace(data.Transforms[0]); - componentTransform.Emplace(data.Transforms[1]); - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } + + TArray componentTransform; + componentTransform.Emplace(data.Transforms[0]); + componentTransform.Emplace(data.Transforms[1]); + // to ensure grounding in the capsule, calculates lowest Z in component space. Does not include hands + for (int32 j = 2; j < parentIndices.Num(); j++) { + componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); + if (j == rShinJoint + 1 || j == rShinJoint + 2 || j == lShinJoint + 1 || j == lShinJoint + 2) + minFeetZ = FGenericPlatformMath::Min(minFeetZ, componentTransform[j].GetTranslation().Z); + else if (j == rShinJoint || j == lShinJoint) + minKnees = FGenericPlatformMath::Min(minKnees, componentTransform[j].GetTranslation().Z); + else if (j >= numBodyJoints) + minHands = FGenericPlatformMath::Min(minHands, componentTransform[j].GetTranslation().Z); + else + minTorso = FGenericPlatformMath::Min(minTorso, componentTransform[j].GetTranslation().Z); } + + float minZ = FMath::Min(FMath::Min(minTorso, minHands), FMath::Min(minFeetZ, minKnees)); + minZ -= rootHipOffsetZ; // assigns motion either to root or to hips @@ -292,11 +308,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -384,7 +404,7 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject data.Transforms.Add(transform); } - AssignCharacterMotion(data); + AssignCharacterMotion(data); CachePose(data.Transforms); hasProcessedRotations = true; @@ -429,6 +449,15 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -619,6 +648,83 @@ void PoseAIRigMixamo::Configure() rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -712,7 +818,11 @@ void PoseAIRigMetaHuman::Configure() void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 4; + lShinJoint = 9; + lowerBodyNumOfJoints = 10; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index 595a509..28e63ac 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -85,6 +85,10 @@ int32 FPoseAIHandshake::GetHandModelVersion() const { return static_cast(handModelVersion) + 1; } +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + FString FPoseAIHandshake::GetModeString() const { switch (mode) { case EPoseAiAppModes::Room: return "Room"; @@ -101,6 +105,7 @@ FString FPoseAIHandshake::GetRigString() const { case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; case EPoseAiRigPresets::UE4: return "UE4"; case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; case EPoseAiRigPresets::DazUE: return "DazUE"; default: return "MetaHuman"; @@ -117,24 +122,30 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), *(GetRigString()), *(GetModeString()), + *(YesNoString(isFaceAnimating)), *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, + GetBodyModelVersion(), GetHandModelVersion(), + *(YesNoString(locomotionEvents)), static_cast(packetFormat) ); } diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index fdf95f7..7403779 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -141,6 +141,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -268,6 +272,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -303,16 +309,22 @@ GENERATED_BODY() /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum = 8080, bool isIPv6 = false); + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..7029303 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,107 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h index 854c15d..be7e999 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -18,6 +18,7 @@ #include "Json.h" #include "PoseAIRig.h" #include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -72,6 +73,8 @@ class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource ILiveLinkClient* liveLinkClient = nullptr; FCriticalSection InSynchObject; FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + mutable FText status; }; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index b959395..f11d9a4 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -19,7 +19,7 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" - +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -102,9 +102,10 @@ class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - + TUniquePtr faceSubSource; mutable FText status; - + FCriticalSection InSynchObject; + void AddSubject(); }; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 1bfa6ba..5aa5894 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -64,12 +64,16 @@ class POSEAILIVELINK_API PoseAIRig bool useRootMotion; bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; + bool isCrouching = false; int32 handZoneL = 5; @@ -89,7 +93,7 @@ class POSEAILIVELINK_API PoseAIRig float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); @@ -102,6 +106,7 @@ class POSEAILIVELINK_API PoseAIRig void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +123,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index ffd1cb8..2f7ee9e 100644 --- a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -40,13 +40,18 @@ enum class EPoseAiContext : uint8 UENUM(BlueprintType) enum class EPoseAiRigPresets : uint8 { - MetaHuman, UE4, Mixamo, DazUE + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt }; UENUM(BlueprintType) enum class EPoseAiHandModel : uint8 { Version1, Version2_EXPERIMENTAL }; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; @@ -64,45 +69,58 @@ struct POSEAILIVELINK_API FPoseAIHandshake UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = true; + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; - + bool useRootMotion = false; /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 cameraFPS = 60; - /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 syncFPS = 0; - - /* controls compactness of packet. */ + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; - + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; - /* the model context. Will enable new AI models as they are deployed*/ + /* the model context. Reserved for future AI models*/ UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } @@ -111,6 +129,7 @@ struct POSEAILIVELINK_API FPoseAIHandshake FString GetContextString() const; FString GetModeString() const; FString GetRigString() const; + int32 GetBodyModelVersion() const; int32 GetHandModelVersion() const; FString ToString() const; FString YesNoString(bool val) const { diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/PoseAILiveLink.uplugin index 9d5a7a8..81a7164 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,7 +1,7 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.40", + "VersionName": "1.42", "FriendlyName": "PoseAI LiveLink", "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 6b53b2d..61106be 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -93,6 +93,9 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -143,25 +146,29 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { -bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { portNum = PoseAILiveLinkNetworkSource::portDefault; while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; } - return AddSource(handshake, myIP, portNum, isIPv6); + return AddSource(handshake, isIPv6, portNum, myIP, subject); } -bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { - FLiveLinkSubjectName addedSubjectName; +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); } + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -210,7 +217,9 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..f68e3da --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + + if (jsonPose != nullptr && jsonPose->HasTypedField("Face")) { + + + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp index 58ac335..61c68f1 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -6,9 +6,6 @@ -FCriticalSection critSingleSectionLocal; - - /* First use the static method to create a source and add it to the LiveLinkClient. * The LiveLinkClient must own only shared pointer or UE will crash on cleanup. * The client will respond with receive client when it is registered. We return a weak ptr @@ -51,6 +48,9 @@ void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid sourceGuid = InSourceGuid; subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); } /* @@ -113,6 +113,7 @@ bool PoseAILiveLinkNativeSource::RequestSourceShutdown() { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; @@ -150,7 +151,8 @@ void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); - + faceSubSource->UpdateFace(jsonPose); } } } + diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp index ff08b49..b18a1a0 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -4,7 +4,6 @@ #include "Features/IModularFeatures.h" #include "PoseAIEventDispatcher.h" -FCriticalSection critSingleSection; /* @@ -89,6 +88,9 @@ void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid liveLinkClient = InClient; AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + } /* @@ -119,7 +121,7 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ subject.Settings = nullptr; subject.VirtualSubject = nullptr; - critSingleSection.Lock(); + FScopeLock ScopeLock(&InSynchObject); if (!liveLinkClient->CreateSubject(subject)) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); } @@ -128,7 +130,6 @@ void PoseAILiveLinkNetworkSource::AddSubject(){ FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); } - critSingleSection.Unlock(); } @@ -145,6 +146,7 @@ void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) data.Transforms.Reserve(100); if (rig->ProcessFrame(jsonPose, data)) { liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); } else { static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; @@ -212,6 +214,7 @@ bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() usedPorts.Remove(port); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); liveLinkClient->RemoveSubject_AnyThread(subjectKey); liveLinkClient->RemoveSource(sourceGuid); liveLinkClient = nullptr; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 5989de6..8d2646d 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -24,14 +24,14 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), rigType(FName(handshake.GetRigString())), - useRootMotion(handshake.useRootMotion), + useRootMotion(handshake.useRootMotion), includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; switch (handshake.rig) { @@ -41,6 +41,9 @@ TSharedPtr PoseAIRig::PoseAIRigFactory(const FLi case EPoseAiRigPresets::Mixamo: rigPtr = MakeShared(name, handshake); break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; case EPoseAiRigPresets::DazUE: rigPtr = MakeShared(name, handshake); break; @@ -117,7 +120,7 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink liveValues.rootTranslation = FVector( -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip 0.0f, //currently no body distance estimate from pose camera - 0.0f + 0.0f // could do this if game calibrates from a player starting position: liveValues.hipScreen[1] * rigHeight / liveValues.bodyHeight ); TriggerEvents(); @@ -223,21 +226,34 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { + FVector baseTranslation = FVector::ZeroVector; + // seperating into lowest body part in case we want to connect with AI later + float minTorso = 0.0f; + float minFeetZ = 0.0f; + float minKnees = 0.0f; + float minHands = 0.0f; + + if (!isDesktop) baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - TArray componentTransform; - componentTransform.Emplace(data.Transforms[0]); - componentTransform.Emplace(data.Transforms[1]); - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } + + TArray componentTransform; + componentTransform.Emplace(data.Transforms[0]); + componentTransform.Emplace(data.Transforms[1]); + // to ensure grounding in the capsule, calculates lowest Z in component space. Does not include hands + for (int32 j = 2; j < parentIndices.Num(); j++) { + componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); + if (j == rShinJoint + 1 || j == rShinJoint + 2 || j == lShinJoint + 1 || j == lShinJoint + 2) + minFeetZ = FGenericPlatformMath::Min(minFeetZ, componentTransform[j].GetTranslation().Z); + else if (j == rShinJoint || j == lShinJoint) + minKnees = FGenericPlatformMath::Min(minKnees, componentTransform[j].GetTranslation().Z); + else if (j >= numBodyJoints) + minHands = FGenericPlatformMath::Min(minHands, componentTransform[j].GetTranslation().Z); + else + minTorso = FGenericPlatformMath::Min(minTorso, componentTransform[j].GetTranslation().Z); } + + float minZ = FMath::Min(FMath::Min(minTorso, minHands), FMath::Min(minFeetZ, minKnees)); + minZ -= rootHipOffsetZ; // assigns motion either to root or to hips @@ -292,11 +308,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -384,7 +404,7 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject data.Transforms.Add(transform); } - AssignCharacterMotion(data); + AssignCharacterMotion(data); CachePose(data.Transforms); hasProcessedRotations = true; @@ -429,6 +449,15 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -619,6 +648,83 @@ void PoseAIRigMixamo::Configure() rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -712,7 +818,11 @@ void PoseAIRigMetaHuman::Configure() void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 4; + lShinJoint = 9; + lowerBodyNumOfJoints = 10; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index 595a509..28e63ac 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -85,6 +85,10 @@ int32 FPoseAIHandshake::GetHandModelVersion() const { return static_cast(handModelVersion) + 1; } +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + FString FPoseAIHandshake::GetModeString() const { switch (mode) { case EPoseAiAppModes::Room: return "Room"; @@ -101,6 +105,7 @@ FString FPoseAIHandshake::GetRigString() const { case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; case EPoseAiRigPresets::UE4: return "UE4"; case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; case EPoseAiRigPresets::DazUE: return "DazUE"; default: return "MetaHuman"; @@ -117,24 +122,30 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), *(GetRigString()), *(GetModeString()), + *(YesNoString(isFaceAnimating)), *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, + GetBodyModelVersion(), GetHandModelVersion(), + *(YesNoString(locomotionEvents)), static_cast(packetFormat) ); } diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index fdf95f7..7403779 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -141,6 +141,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -268,6 +272,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -303,16 +309,22 @@ GENERATED_BODY() /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum = 8080, bool isIPv6 = false); + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") - bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..7029303 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,107 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h index 854c15d..be7e999 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -18,6 +18,7 @@ #include "Json.h" #include "PoseAIRig.h" #include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -72,6 +73,8 @@ class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource ILiveLinkClient* liveLinkClient = nullptr; FCriticalSection InSynchObject; FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + mutable FText status; }; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index b959395..f11d9a4 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -19,7 +19,7 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" - +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -102,9 +102,10 @@ class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - + TUniquePtr faceSubSource; mutable FText status; - + FCriticalSection InSynchObject; + void AddSubject(); }; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 1bfa6ba..5aa5894 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -64,12 +64,16 @@ class POSEAILIVELINK_API PoseAIRig bool useRootMotion; bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; + bool isCrouching = false; int32 handZoneL = 5; @@ -89,7 +93,7 @@ class POSEAILIVELINK_API PoseAIRig float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); @@ -102,6 +106,7 @@ class POSEAILIVELINK_API PoseAIRig void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +123,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index ffd1cb8..2f7ee9e 100644 --- a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -40,13 +40,18 @@ enum class EPoseAiContext : uint8 UENUM(BlueprintType) enum class EPoseAiRigPresets : uint8 { - MetaHuman, UE4, Mixamo, DazUE + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt }; UENUM(BlueprintType) enum class EPoseAiHandModel : uint8 { Version1, Version2_EXPERIMENTAL }; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; @@ -64,45 +69,58 @@ struct POSEAILIVELINK_API FPoseAIHandshake UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = true; + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; - + bool useRootMotion = false; /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 cameraFPS = 60; - /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") int32 syncFPS = 0; - - /* controls compactness of packet. */ + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; - + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; - /* the model context. Will enable new AI models as they are deployed*/ + /* the model context. Reserved for future AI models*/ UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } @@ -111,6 +129,7 @@ struct POSEAILIVELINK_API FPoseAIHandshake FString GetContextString() const; FString GetModeString() const; FString GetRigString() const; + int32 GetBodyModelVersion() const; int32 GetHandModelVersion() const; FString ToString() const; FString YesNoString(bool val) const { diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/PoseAILiveLink.uplugin similarity index 68% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/PoseAILiveLink.uplugin rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/PoseAILiveLink.uplugin index ba0d8e8..5c70adb 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,28 +1,33 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.33", + "VersionName": "1.42", "FriendlyName": "PoseAI LiveLink", - "Description": "Live Link plugin to stream from the Pose Camera mobile app by PoseAI", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", "CreatedBy": "Pose AI Ltd", "CreatedByURL": "www.pose-ai.com", "DocsURL": "www.pose-ai.com/unreal-engine-livelink", "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/3db8f23ad003492eb85a50de31e5bc57", "SupportURL": "support@pose-ai.com", - "EngineVersion": "4.25.0", + "EngineVersion": "5.2.0", "CanContainContent": false, "IsBetaVersion": false, - "IsExperimentalVersion": false, - "Installed": false, + "Installed": true, "Modules": [ { "Name": "PoseAILiveLink", "Type": "Runtime", "LoadingPhase": "Default", "WhitelistPlatforms": [ - "Win64", "Mac" + "Win64", + "Mac" ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" } ], "Plugins": [ @@ -31,5 +36,4 @@ "Enabled": true } ] - } \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Resources/Icon128.png similarity index 100% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Resources/Icon128.png rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Resources/Icon128.png diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs similarity index 71% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs index da090b0..2e1e945 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Epic Games, Inc. All Rights Reserved. using UnrealBuildTool; @@ -33,32 +33,41 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) "Networking", "Sockets", "LiveLink", - "LiveLinkAnimationCore", "LiveLinkInterface", "Json", "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", // ... add other public dependencies that you statically link with here ... } ); - - + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + PrivateDependencyModuleNames.AddRange( new string[] { "Slate", "SlateCore", + // ... add private dependencies that you statically link with here ... } ); - - + + DynamicallyLoadedModuleNames.AddRange( new string[] { // ... add any modules that your module loads dynamically here ... } ); - // PublicDefinitions.Add("WINDOWS_IGNORE_PACKING_MISMATCH"); - } } diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..dfb800c --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,105 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp similarity index 79% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp index ccaf3bc..7e3ec70 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -1,6 +1,7 @@ -// Copyright 2021 Pose AI Ltd. . +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEndpoint.h" +#define LOCTEXT_NAMESPACE "PoseAI" ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; @@ -9,7 +10,11 @@ TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int FName socketType = NAME_DGram; FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); - socket->SetNonBlocking(); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + socket->SetReuseAddr(); int actualSize; socket->SetReceiveBufferSize(64 * 1024, actualSize); @@ -34,4 +39,5 @@ void FPoseAIEndpoint::Initialize() CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); } +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp similarity index 89% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 7189e64..a749d8f 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -1,7 +1,8 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" +#define LOCTEXT_NAMESPACE "PoseAI" void UStepCounter::Halt(bool fade) { num_ = 0; @@ -76,8 +77,8 @@ UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { - portNum = PoseAILiveLinkSingleSource::portDefault; - while (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; @@ -89,10 +90,13 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr InitializeObjects(); FLiveLinkSubjectName addedSubjectName; PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkSingleSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -111,7 +115,7 @@ bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeI InitializeObjects(); bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); if (success) - onRegistered.Broadcast(name, PoseAILiveLinkSingleSource::GetConnectionName(name)); + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); return success; } @@ -142,9 +146,30 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { } + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -189,11 +214,13 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam UPoseAIMovementComponent* existing_component; bool isReconnection = HasComponent(subjectName, existing_component); if (isReconnection) { - if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkSingleSource::GetConnectionName(subjectName)); + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); @@ -217,7 +244,8 @@ void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { } void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { - knownConnectionsWithTime.Add(subjectName,FDateTime::Now()); + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { UPoseAIMovementComponent* component; if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); @@ -384,4 +412,6 @@ bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPos return false; component = componentsByName[name]; return component != nullptr && IsValid(component); -} \ No newline at end of file +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..6717a12 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,19 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..d3a21f8 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,117 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + + if (jsonPose != nullptr && jsonPose->HasTypedField("Face")) { + + + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..f552358 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,170 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..2735906 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp index 63f5013..1284488 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkRetargetRotations.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp similarity index 75% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp index 8ac9271..c93d284 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -1,21 +1,27 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkServer.h" #include "Async/Async.h" #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" +#define LOCTEXT_NAMESPACE "PoseAI" -const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("0.8.24")); + +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); -PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6, int32 portNum) : - source_(mySource), handshake(myHandshake), port(portNum) { +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); @@ -23,7 +29,7 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); serverSocket = BuildUdpSocket(serverName, protocolType, port); - poseAILiveLinkRunnable = MakeShared(port, this); + poseAILiveLinkRunnable = MakeShared(port, listener, this); udpSocketSender = MakeShared(serverSocket, *senderName); FString myIP; @@ -36,6 +42,10 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL } } +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + bool PoseAILiveLinkServer::GetIP(FString& myIP) { bool canBind = false; @@ -51,7 +61,6 @@ bool PoseAILiveLinkServer::GetIP(FString& myIP) { } void PoseAILiveLinkServer::CleanUp() { - source_ = nullptr; if (!cleaningUp) { cleaningUp = true; CleanUpReceiver(); @@ -68,18 +77,18 @@ void PoseAILiveLinkServer::CleanUp() { } void PoseAILiveLinkServer::CleanUpReceiver() { - if (udpSocketReceiver != nullptr && udpSocketReceiver.IsValid()) { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); udpSocketReceiver->Stop(); } - if (poseAILiveLinkRunnable != nullptr && poseAILiveLinkRunnable.IsValid()) { + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); poseAILiveLinkRunnable.Reset(); } } void PoseAILiveLinkServer::CleanUpSender() { - if (udpSocketSender != nullptr && udpSocketSender.IsValid()) { + if (udpSocketSender && udpSocketSender.IsValid()) { Disconnect(); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); udpSocketSender->Stop(); @@ -92,7 +101,7 @@ bool PoseAILiveLinkServer::HasValidConnection() const { return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; } -void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { static const FGuid GUID_Error = FGuid(); if (cleaningUp) return; @@ -102,12 +111,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); - FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s"), *endpointRecv.ToString()); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); return; } + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); if (HasValidConnection() && !sameAsCurrent) { - if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint SendHandshake(); } @@ -115,7 +125,6 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); //consider sending rejected connection a warning message } - } else if (!HasValidConnection() && !sameAsCurrent) { // new connection InitiateConnection(jsonObject, endpointRecv); @@ -124,10 +133,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const else { if (PoseAIRig::IsFrameData(jsonObject)) { lastConnection = FDateTime::Now(); - source_->UpdatePose(rig, jsonObject); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(source_->GetSubjectName()); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } } - else if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { //is likely a repeat hello message + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message SendHandshake(); } } @@ -150,30 +162,37 @@ void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject return; } FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); - source_->SetConnectionName(connectionName); - endpoint = endpointRecv; - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - SendHandshake(); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_->GetSubjectName()); - lastConnection = FDateTime::Now(); UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } } bool PoseAILiveLinkServer::SendString(FString& message) const { - if (!endpoint.IsValid()) + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { return false; - FTCHARToUTF8 byteConvert(*message); - TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); - bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; - return udpSocketSender->Send(bytedata, endpoint); + } } + void PoseAILiveLinkServer::SendHandshake() const { FString message_string = handshake.ToString(); if (SendString(message_string)) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); - } else { //unsuccesful + } else { //unsuccessful static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; static const FGuid GUID_HandshakeFail = FGuid(); FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); @@ -182,31 +201,12 @@ void PoseAILiveLinkServer::SendHandshake() const { } void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { - bool dirty = handshake != newHandshake; - bool rigChange = handshake.rig != newHandshake.rig; handshake = newHandshake; - if (rigChange) { - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - } - if (dirty && endpoint.IsValid()) + if (endpoint.IsValid()) SendHandshake(); } -void PoseAILiveLinkServer::SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { - if (target == source_->GetSubjectName()) { - FString message_string = config.ToString(); - if (SendString(message_string)) - UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s to %s"), *message_string, *endpoint.ToString()); - } -} - -void PoseAILiveLinkServer::CloseTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - source_->RequestSourceShutdown(); -} - void PoseAILiveLinkServer::Disconnect() { if (endpoint.IsValid()) { @@ -222,10 +222,6 @@ void PoseAILiveLinkServer::Disconnect() { } } -void PoseAILiveLinkServer::DisconnectTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - Disconnect(); -} FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const { @@ -253,3 +249,19 @@ bool PoseAILiveLinkServer::CheckAppVersion(FString version) const } return true; } + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp similarity index 91% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp index 24ee263..c7b1432 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkSourceFactory.h" #include "SPoseAILiveLinkWidget.h" diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp new file mode 100644 index 0000000..97bf5a5 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -0,0 +1,920 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +const FString PoseAIRig::fieldBody = FString(TEXT("Body")); +const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); +const FString PoseAIRig::fieldHandLeft = FString(TEXT("LeftHand")); +const FString PoseAIRig::fieldHandRight = FString(TEXT("RightHand")); +const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); +const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); +const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); +const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); + + +bool isDifferentAndSet(int32 newValue, int32& storedValue) { + bool isDifferent = newValue != storedValue; + storedValue = newValue; + return isDifferent; +} + + +PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : + name(name), + rigType(FName(handshake.GetRigString())), + useRootMotion(handshake.useRootMotion), + includeHands(handshake.IncludesHands()), + isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { + Configure(); +} + +TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { + TSharedPtr rigPtr; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; + } + + rigPtr->Configure(); + return rigPtr; +} + +FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ + FLiveLinkStaticDataStruct staticData; + staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); + FLiveLinkSkeletonStaticData* skelData = staticData.Cast(); + check(skelData); + skelData->SetBoneNames(jointNames); + skelData->SetBoneParents(parentIndices); + return staticData; +} + +void PoseAIRig::AddBone(FName boneName, FName parentName, FVector translation) { + jointNames.Emplace(boneName); + if (parentName == boneName) { + parentIndices.Emplace(-1); + } else { + parentIndices.Emplace(jointNames.IndexOfByKey(parentName)); + } + boneVectors.Add(boneName,translation); + lastBoneAdded = boneName; +} + +void PoseAIRig::AddBoneToLast(FName boneName, FVector translation) { + AddBone(boneName, lastBoneAdded, translation); +} + +bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) +{ + return (jsonObject->HasField(fieldBody)) || (jsonObject->HasField(fieldHandLeft)) || (jsonObject->HasField(fieldHandRight)); +} + +bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + + double timestamp; + jsonObject->TryGetNumberField("Timestamp", timestamp); + // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. + if (liveValues.timestamp - 600.0 < timestamp && timestamp < liveValues.timestamp) { + return false; + } + liveValues.timestamp = timestamp; + + FString rigStringOut; + if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } + return false; + } + + uint32 packetFormat = 0; + jsonObject->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 1) + ProcessCompactSupplementaryData(jsonObject, data); + else + ProcessVerboseSupplementaryData(jsonObject, data); + + if (visibilityFlags.isTorso && liveValues.bodyHeight > 0.0f) + liveValues.rootTranslation = FVector( + liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight * ((isMirrored) ? 1.0f : -1.0f), //x is left in Unreal so unmirrored is flipped + 0.0f, //currently no body distance estimate from pose camera + 0.0f // could do this if game calibrates from a player starting position: liveValues.hipScreen[1] * rigHeight / liveValues.bodyHeight + ); + + TriggerEvents(); + + data.WorldTime = FPlatformTime::Seconds(); + bool has_processed = (packetFormat == 1) ? ProcessCompactRotations(jsonObject, data) : ProcessVerboseRotations(jsonObject, data); + return has_processed; +} + +void PoseAIRig::TriggerEvents() { + /* trigger various events and update the Pose AI Movement Component */ + if (visibilityFlags.HasChanged()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastVisibilityChange(name, visibilityFlags); + } + if (verbose.Events.Jump.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastJumps(name); + } + if (verbose.Events.Footstep.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.Footstep.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFootsteps(name, height, verbose.Events.Footstep.Magnitude > 0.0f); + } + if (verbose.Events.FeetSplit.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.FeetSplit.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFeetsplits(name, width, verbose.Events.FeetSplit.Magnitude < 0.0f); + } + if (verbose.Events.ArmPump.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.ArmPump.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmpumps(name, height); + } + if (verbose.Events.ArmFlex.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.ArmFlex.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmflexes(name, width, verbose.Events.ArmFlex.Magnitude < 0.0f); + } + if (verbose.Events.SidestepL.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepL(name, verbose.Events.SidestepL.Magnitude < 0.0f); + } + if (verbose.Events.SidestepR.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepR(name, verbose.Events.SidestepR.Magnitude < 0.0f); + } + if (verbose.Events.ArmGestureL.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureL.Current == 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, true); + else if (verbose.Events.ArmGestureL.Current == 51) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, false); + else + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureL(name, verbose.Events.ArmGestureL.Current); + } + if (verbose.Events.ArmGestureR.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureR.Current < 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureR(name, verbose.Events.ArmGestureR.Current); + } + if (isDifferentAndSet(liveValues.handZoneLeft, handZoneL)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneL(name, handZoneL); + } + if (isDifferentAndSet(liveValues.handZoneRight, handZoneR)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneR(name, handZoneR); + } + if (isDifferentAndSet(liveValues.stableFeet, stableFeet)) { + if (stableFeet > 1) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastStationary(name); + } + if (liveValues.isCrouching != isCrouching) { + isCrouching = !isCrouching; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCrouches(name, isCrouching); + } + if (visibilityFlags.isTorso) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastLiveValues(name, liveValues); +} + + +void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) { + FString VisA = (objBody->HasTypedField("VisA")) ? objBody->GetStringField("VisA") : ""; + FString ScaA = (objBody->HasTypedField("ScaA")) ? objBody->GetStringField("ScaA") : ""; + FString VecA = (objBody->HasTypedField("VecA")) ? objBody->GetStringField("VecA") : ""; + FString EveA = (objBody->HasTypedField("EveA")) ? objBody->GetStringField("EveA") : ""; + visibilityFlags.ProcessCompact(VisA); + liveValues.ProcessCompactScalarsBody(ScaA); + liveValues.ProcessCompactVectorsBody(VecA); + verbose.Events.ProcessCompactBody(EveA); + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + + } + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + FString VecA = (objHandLeft->HasTypedField("VecA")) ? objHandLeft->GetStringField("VecA") : ""; + liveValues.ProcessCompactVectorsHandLeft(VecA); + } + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) { + FString VecA = (objHandRight->HasTypedField("VecA")) ? objHandRight->GetStringField("VecA") : ""; + liveValues.ProcessCompactVectorsHandRight(VecA); + } +} + +void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { + FVector baseTranslation = FVector::ZeroVector; + // seperating into lowest body part in case we want to connect with AI later + float minTorso = 0.0f; + float minFeetZ = 0.0f; + float minKnees = 0.0f; + float minHands = 0.0f; + + if (!isDesktop) + baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame + + TArray componentTransform; + componentTransform.Emplace(data.Transforms[0]); + componentTransform.Emplace(data.Transforms[1]); + // to ensure grounding in the capsule, calculates lowest Z in component space. Does not include hands + for (int32 j = 2; j < parentIndices.Num(); j++) { + componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); + if (j == rShinJoint + 1 || j == rShinJoint + 2 || j == lShinJoint + 1 || j == lShinJoint + 2) + minFeetZ = FGenericPlatformMath::Min(minFeetZ, componentTransform[j].GetTranslation().Z); + else if (j == rShinJoint || j == lShinJoint) + minKnees = FGenericPlatformMath::Min(minKnees, componentTransform[j].GetTranslation().Z); + else if (j >= numBodyJoints) + minHands = FGenericPlatformMath::Min(minHands, componentTransform[j].GetTranslation().Z); + else + minTorso = FGenericPlatformMath::Min(minTorso, componentTransform[j].GetTranslation().Z); + } + + float minZ = FMath::Min(FMath::Min(minTorso, minHands), FMath::Min(minFeetZ, minKnees)); + + minZ -= rootHipOffsetZ; + + // assigns motion either to root or to hips + if (useRootMotion) { + //hip to low point Z distance assigned to hips, rest of movement assigned to root + data.Transforms[1].SetTranslation(FVector(0.0f, 0.0f, -minZ)); + data.Transforms[0].SetTranslation(baseTranslation); + } + else { + baseTranslation.Z -= minZ; + data.Transforms[1].SetTranslation(baseTranslation); + } +} + +bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + FString rotaBody; + FString rotaHandLeft; + FString rotaHandRight; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotaBody = (objBody->HasTypedField("RotA")) ? objBody->GetStringField("RotA") : ""; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotaHandLeft = (objHandLeft->HasTypedField("RotA")) ? objHandLeft->GetStringField("RotA") : ""; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotaHandRight = (objHandRight->HasTypedField("RotA")) ? objHandRight->GetStringField("RotA") : ""; + + + bool hasProcessedRotations; + + if ((rotaBody.Len() < 8 && cachedPose.Num() < 1) ) { + hasProcessedRotations = false; + } + else if (rotaBody.Len() < 8 ) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + AppendCachedRotations(0, 1, componentRotations, data); + + if (rotaBody.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaBody, flatArray); + FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint + } + else + AppendCachedRotations(1, numBodyJoints, componentRotations, data); + + + if (includeHands) { + if (rotaHandLeft.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandLeft, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints, numBodyJoints + numHandJoints, componentRotations, data); + if (rotaHandRight.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandRight, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints + numHandJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints + numHandJoints, numBodyJoints + 2 * numHandJoints, componentRotations, data); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > rotBody = nullptr; + TSharedPtr < FJsonObject > rotHandLeft = nullptr; + TSharedPtr < FJsonObject > rotHandRight = nullptr; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotBody = (objBody->HasTypedField(fieldRotations)) ? objBody->GetObjectField(fieldRotations) : nullptr; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotHandLeft = (objHandLeft->HasTypedField(fieldRotations)) ? objHandLeft->GetObjectField(fieldRotations) : nullptr; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotHandRight = (objHandRight->HasTypedField(fieldRotations)) ? objHandRight->GetObjectField(fieldRotations) : nullptr; + + + bool hasProcessedRotations; + if (rotBody == nullptr && cachedPose.Num() < 1) { + hasProcessedRotations = false; + } + else if (rotBody == nullptr || !visibilityFlags.isTorso) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + + for (int32 i = 0; i < jointNames.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + FQuat rotation; + const TArray < TSharedPtr < FJsonValue > >* outArray; + FString jointString = jointName.ToString(); + if ((rotBody != nullptr && rotBody->TryGetArrayField(jointString, outArray)) || + (rotHandLeft != nullptr && rotHandLeft->TryGetArrayField(jointString, outArray)) || + (rotHandRight != nullptr && rotHandRight->TryGetArrayField(jointString, outArray))) { + rotation = FQuat((*outArray)[0]->AsNumber(), (*outArray)[1]->AsNumber(), (*outArray)[2]->AsNumber(), (*outArray)[3]->AsNumber()); + } + else if (cachedPose.Num() > i) { + rotation = parentQuat * cachedPose[i].GetRotation(); + } + else { + rotation = FQuat::Identity; + } + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations + data.Transforms.Add(transform); + } + + AssignCharacterMotion(data); + CachePose(data.Transforms); + + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > scaBody = nullptr; + TSharedPtr < FJsonObject > eveBody = nullptr; + TSharedPtr < FJsonObject > vecBody = nullptr; + TSharedPtr < FJsonObject > vecHandLeft = nullptr; + TSharedPtr < FJsonObject > vecHandRight = nullptr; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + + if (objBody != nullptr && objBody.IsValid()) { + verbose.ProcessJsonObject(objBody); + //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; + } + + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; + + } + if (objHandRight != nullptr && objHandRight.IsValid()) { + vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + } + + liveValues.ProcessVerboseBody(verbose); + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + liveValues.ProcesssVerboseVectorsHandRight(vecHandRight); + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + visibilityFlags.ProcessVerbose(verbose.Scalars); +} + +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + +void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < begin + quatArray.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + const FQuat& rotation = quatArray[i - begin]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < end; i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FQuat& rotation = (cachedPose.Num() > i) ? parentQuat * cachedPose[i].GetRotation() : FQuat::Identity; + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::CachePose(const TArray& transforms) { + //cachedPose.Empty(); + cachedPose = TArray(transforms); +} + +void PoseAIRig::Configure() {} + +void PoseAIRigUE4::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-1.448829, 0.531424, 9.00581)); + AddBoneToLast(TEXT("calf_r"), FVector(42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(10.453837, -16.577854, 0.080156)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-1.448829, 0.531424, -9.00581)); + AddBoneToLast(TEXT("calf_l"), FVector(-42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-10.453837, 16.577854, 0.080156)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(10.808878, 0.851415, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(18.875349, -3.801159, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(13.407329, -0.420477, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(16.558783, 0.355318, 0)); + AddBoneToLast(TEXT("head"), FVector(9.283613, -0.364157, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_03"), FVector(11.883688, 2.732088, -3.781983)); + AddBoneToLast(TEXT("upperarm_l"), FVector(15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(30.33993, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_03"), FVector(11.883688, 2.732102, 3.782003)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-30.33993, 0, 0)); + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("index_01_l"), TEXT("hand_l"), FVector(12.068114, -1.763462, -2.109398)); + AddBoneToLast(TEXT("index_02_l"), FVector(4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(3.39379, 0, 0)); + AddBone(TEXT("middle_01_l"), TEXT("hand_l"), FVector(12.244281, -1.293644, 0.571162)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(3.648844, 0, 0)); + AddBone(TEXT("ring_01_l"), TEXT("hand_l"), FVector(11.497885, -1.753527, 2.846912)); + AddBoneToLast(TEXT("ring_02_l"), FVector(4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(3.476652, 0, 0)); + AddBone(TEXT("pinky_01_l"), TEXT("hand_l"), FVector(10.140665, -2.263151, 4.643148)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(2.985631, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(4.762036, -2.374981, -2.53782)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(4.062171, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("index_01_r"), TEXT("hand_r"), FVector(-12.068114, 1.763462, 2.109398)); + AddBoneToLast(TEXT("index_02_r"), FVector(-4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-3.39379, 0, 0)); + AddBone(TEXT("middle_01_r"), TEXT("hand_r"), FVector(-12.244281, 1.293644, -0.571162)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-3.648844, 0, 0)); + AddBone(TEXT("ring_01_r"), TEXT("hand_r"), FVector(-11.497885, 1.753527, -2.846912)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-3.476652, 0, 0)); + AddBone(TEXT("pinky_01_r"), TEXT("hand_r"), FVector(-10.140665, 2.263151, -4.643148)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-2.985631, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-4.762036, 2.374981, 2.53782)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-4.062171, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + + rigHeight = 170.0f; + rig = MakeStaticData(); +} + + +void PoseAIRigMixamo::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, -44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(-0.7, -17.8, -5.8)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, -44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0.7, -17.8, -5.8)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(0, -25.7, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("RightForeArm"), FVector(0, -25.7, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(0, -14.0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(0, -14.0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + + + +void PoseAIRigMetaHuman::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-2.3, 0.4, 9.27)); + AddBoneToLast(TEXT("calf_r"), FVector(41.2, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.0, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(7.1, -14.4, -0.4)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-2.3, 0.4, -9.27)); + AddBoneToLast(TEXT("calf_l"), FVector(-41.2, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.0, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-7.1, 14.4, 0.4)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(3.4, 0.0, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(6.3, 0.0, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(6.9, 0.0, 0)); + AddBoneToLast(TEXT("spine_04"), FVector(8.1, 0.0, 0)); + AddBoneToLast(TEXT("spine_05"), FVector(18.3, 0.0, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(11.6, 1.0, 0)); + AddBoneToLast(TEXT("neck_02"), FVector(5.0, 0, 0)); + AddBoneToLast(TEXT("head"), FVector(5.0, 0, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_05"), FVector(5.5, -0.7, -1.2)); + AddBoneToLast(TEXT("upperarm_l"), FVector(17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(27.0, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_05"), FVector(5.5, -0.7, 1.2)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-27.0, 0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_l"), TEXT("lowerarm_l"), FVector(7.0, 0, 0)); + + AddBone(TEXT("index_metacarpal_l"), TEXT("hand_l"), FVector(3.5, 0.4, -2.1)); + AddBoneToLast(TEXT("index_01_l"), FVector(5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_l"), TEXT("hand_l"), FVector(3.3, 0.3, -0.1)); + AddBoneToLast(TEXT("middle_01_l"), FVector(6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_l"), TEXT("hand_l"), FVector(3.2, -0.2, 1.2)); + AddBoneToLast(TEXT("ring_01_l"), FVector(6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_l"), TEXT("hand_l"), FVector(3.1, -0.7, 2.4)); + AddBoneToLast(TEXT("pinky_01_l"), FVector(5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(1.8, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(2.0, -1.0, -2.6)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(2.7, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_r"), TEXT("lowerarm_r"), FVector(-7.0, 0, 0)); + AddBone(TEXT("index_metacarpal_r"), TEXT("hand_r"), FVector(-3.5, -0.4, 2.1)); + AddBoneToLast(TEXT("index_01_r"), FVector(-5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_r"), TEXT("hand_r"), FVector(-3.3, -0.3, 0.1)); + AddBoneToLast(TEXT("middle_01_r"), FVector(-6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_r"), TEXT("hand_r"), FVector(-3.2, 0.2, -1.2)); + AddBoneToLast(TEXT("ring_01_r"), FVector(-6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_r"), TEXT("hand_r"), FVector(-3.1, 0.7, -2.4)); + AddBoneToLast(TEXT("pinky_01_r"), FVector(-5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-1.8, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-2.0, 1.0, 2.6)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 168.0f; + rig = MakeStaticData(); +} + +void PoseAIRigDazUE::Configure() +{ + //daz has extra joints in the legs + rShinJoint = 4; + lShinJoint = 9; + lowerBodyNumOfJoints = 10; + + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hip"), FVector(0.0, -105.0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, -1.8, 0.0)); + + AddBone(TEXT("rThighBend"), TEXT("pelvis"), FVector(-7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("rThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("rShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("rFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("rToe"), FVector(0.0, 0, 14.0)); + + AddBone(TEXT("lThighBend"), TEXT("pelvis"), FVector(7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("lThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("lShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("lFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("lToe"), FVector(0.0, 0.0, 14.0)); + + AddBone(TEXT("abdomenLower"), TEXT("hip"), FVector(0.0, -1.7, -1.5)); + AddBoneToLast(TEXT("abdomenUpper"), FVector(0.0, -8.2, 1.2)); + AddBoneToLast(TEXT("chestLower"), FVector(0.0, -7.9, -0.4)); + AddBoneToLast(TEXT("chestUpper"), FVector(0.0, -13.1, -3.6)); + AddBoneToLast(TEXT("neckLower"), FVector(0.0, -18.3, -1.5)); + AddBoneToLast(TEXT("neckUpper"), FVector(0.0, -3.5, 1.5)); + AddBoneToLast(TEXT("head"), FVector(0.0, -4.9, -0.5)); + + AddBone(TEXT("lCollar"), TEXT("chestUpper"), FVector(3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("lShldrBend"), FVector(11.9, 1.7, 0)); + AddBoneToLast(TEXT("lShldrTwist"), FVector(11.6, 0, 0)); + AddBoneToLast(TEXT("lForearmBend"), FVector(14.4, -0.2, -0.5)); + + AddBone(TEXT("rCollar"), TEXT("chestUpper"), FVector(-3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("rShldrBend"), FVector(-11.9, 1.7, 0)); + AddBoneToLast(TEXT("rShldrTwist"), FVector(-11.6, 0, 0)); + AddBoneToLast(TEXT("rForearmBend"), FVector(-14.4, -0.2, -0.5)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("lForearmTwist"), TEXT("lForearmBend"), FVector(12.1, 0, 0)); + AddBone(TEXT("lHand"), TEXT("lForearmTwist"), FVector(14.2, 0, -0.3)); + + AddBone(TEXT("lCarpal1"), TEXT("lHand"), FVector(0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("lIndex1"), FVector(7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("lIndex2"), FVector(3.9, 0, 0)); + AddBoneToLast(TEXT("lIndex3"), FVector(2.1, 0, 0)); + AddBone(TEXT("lCarpal2"), TEXT("lHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("lMid1"), FVector(7.5, -0.3, 0)); + AddBoneToLast(TEXT("lMid2"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("lMid3"), FVector(2.5, 0, 0)); + AddBone(TEXT("lCarpal3"), TEXT("lHand"), FVector(0.8, -0.4, -0.8)); + AddBoneToLast(TEXT("lRing1"), FVector(6.9, -0.2, 0.0)); + AddBoneToLast(TEXT("lRing2"), FVector(4.0, 0, 0)); + AddBoneToLast(TEXT("lRing3"), FVector(2.2, 0, 0)); + AddBone(TEXT("lCarpal4"), TEXT("lHand"), FVector(0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("lPinky1"), FVector(6.5, 0.2, 0)); + AddBoneToLast(TEXT("lPinky2"), FVector(2.8, 0, 0)); + AddBoneToLast(TEXT("lPinky3"), FVector(1.7, 0, 0)); + AddBone(TEXT("lThumb1"), TEXT("lHand"), FVector(1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("lThumb2"), FVector(4.1, 0, 0)); + AddBoneToLast(TEXT("lThumb3"), FVector(3.0, 0, 0)); + + AddBone(TEXT("rForearmTwist"), TEXT("rForearmBend"), FVector(-12.1, 0, 0)); + AddBone(TEXT("rHand"), TEXT("rForearmTwist"), FVector(-14.2, 0, -0.3)); + + AddBone(TEXT("rCarpal1"), TEXT("rHand"), FVector(-0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("rIndex1"), FVector(-7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("rIndex2"), FVector(-3.9, 0, 0)); + AddBoneToLast(TEXT("rIndex3"), FVector(-2.1, 0, 0)); + AddBone(TEXT("rCarpal2"), TEXT("rHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("rMid1"), FVector(-7.5, -0.3, 0)); + AddBoneToLast(TEXT("rMid2"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("rMid3"), FVector(-2.5, 0, 0)); + AddBone(TEXT("rCarpal3"), TEXT("rHand"), FVector(-0.8, -0.4, 0.8)); + AddBoneToLast(TEXT("rRing1"), FVector(-6.9, -0.2, 0)); + AddBoneToLast(TEXT("rRing2"), FVector(-4.0, 0, 0)); + AddBoneToLast(TEXT("rRing3"), FVector(-2.2, 0, 0)); + AddBone(TEXT("rCarpal4"), TEXT("rHand"), FVector(-0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("rPinky1"), FVector(-6.5, 0.2, 0)); + AddBoneToLast(TEXT("rPinky2"), FVector(-2.8, 0, 0)); + AddBoneToLast(TEXT("rPinky3"), FVector(-1.7, 0, 0)); + AddBone(TEXT("rThumb1"), TEXT("rHand"), FVector(-1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("rThumb2"), FVector(-4.1, 0, 0)); + AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 168.0f; + rig = MakeStaticData(); +} + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp similarity index 71% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index aa31622..0788c25 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -1,7 +1,12 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIStructs.h" +#define LOCTEXT_NAMESPACE "PoseAI" + + +// Utility conversion functions for compact representation and from arrays to vectors + float UintB64ToUint(char a, char b) { static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; @@ -19,14 +24,12 @@ float FixedB64pairToFloat(char a, char b) { } void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { - check(data.Len() % 2 == 0); flatArray.Reserve(flatArray.Num() + data.Len() / 2); for (int i = 0; i + 1 < data.Len(); i += 2) flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); } void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { - check(flatArray.Num() % 4 == 0); quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); for (int i = 0; i + 3 < flatArray.Num(); i += 4) quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); @@ -35,19 +38,37 @@ void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ const TArray < TSharedPtr < FJsonValue > >* value; - if (jsonObj->TryGetArrayField(fieldName, value)){ + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ fieldVector2D.X = (*value)[0]->AsNumber(); fieldVector2D.Y = (*value)[1]->AsNumber(); } } +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { - if (value.Num() == 2) { + if (value.Num() >= 2) { fieldVector2D.X = value[0]; fieldVector2D.Y = value[1]; } } +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { if (newValue != field) changeFlag = true; @@ -56,7 +77,47 @@ void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { SetAndCheckForChange(newValue > 0.5f, field, changeFlag); } +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} FString FPoseAIHandshake::ToString() const { return FString::Printf( @@ -64,31 +125,42 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), - *rig, - *mode, - *context, + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, - packetFormat + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) ); } + bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const { return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; } + + FString FPoseAIModelConfig::ToString() const { return FString::Printf( TEXT("{\"CONFIG\":{" @@ -157,24 +229,59 @@ void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { if (compactString.Len() < 12) return; - upperBodyLean.Set(FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, - FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f); - hipScreen.Set(FixedB64pairToFloat(compactString[4], compactString[5]), - FixedB64pairToFloat(compactString[6], compactString[7])); - chestScreen.Set(FixedB64pairToFloat(compactString[8], compactString[9]), - FixedB64pairToFloat(compactString[10], compactString[11])); + upperBodyLean.Set( + FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, + FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f + ); + hipScreen.Set( + FixedB64pairToFloat(compactString[4], compactString[5]), + FixedB64pairToFloat(compactString[6], compactString[7]) + ); + chestScreen.Set( + FixedB64pairToFloat(compactString[8], compactString[9]), + FixedB64pairToFloat(compactString[10], compactString[11]) + ); + if (compactString.Len() < 24) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[12], compactString[13]) * 4.0f, + FixedB64pairToFloat(compactString[14], compactString[15]) * 4.0f, + FixedB64pairToFloat(compactString[16], compactString[17]) * 4.0f + ); + handIkR.Set( + FixedB64pairToFloat(compactString[18], compactString[19]) * 4.0f, + FixedB64pairToFloat(compactString[20], compactString[21]) * 4.0f, + FixedB64pairToFloat(compactString[22], compactString[23]) * 4.0f + ); + } void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandLeft.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandLeft.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkL.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } void FPoseAILiveValues::ProcessCompactVectorsHandRight(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandRight.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandRight.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkR.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } @@ -206,7 +313,7 @@ void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ bodyHeight = verbose.Scalars.BodyHeight; - stableFeet = FMath::RoundToInt(verbose.Scalars.StableFoot); + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); stanceYaw = verbose.Scalars.StanceYaw * 180.0f; chestYaw = verbose.Scalars.ChestYaw * 180.0f; isCrouching = verbose.Scalars.IsCrouching > 0.5f; @@ -216,6 +323,8 @@ void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ upperBodyLean *= 180.0f; ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); } @@ -224,11 +333,15 @@ void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonOb if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); + } void FPoseAILiveValues::ProcesssVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); } +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp similarity index 92% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp index ace89a0..ae6882d 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -1,18 +1,19 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "SPoseAILiveLinkWidget.h" #include "PoseAILiveLinkSourceFactory.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" #define LOCTEXT_NAMESPACE "PoseAI" TWeakPtr SPoseAILiveLinkWidget::source = nullptr; +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; -static TArray PoseAI_Rigs = { "UE4", "MetaHuman", "Mixamo", "DazUE"}; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; -int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkSingleSource::portDefault; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; int32 SPoseAILiveLinkWidget::syncFPS = 60; int32 SPoseAILiveLinkWidget::cameraFPS = 60; int32 SPoseAILiveLinkWidget::modeIndex = 0; @@ -192,7 +193,7 @@ bool SPoseAILiveLinkWidget::IsPortValid() const return false; } - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); return false; } @@ -234,22 +235,25 @@ void SPoseAILiveLinkWidget::disableExistingSource() TSharedPtr linkSource = source.Pin(); if (linkSource.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); - ((PoseAILiveLinkSingleSource*)linkSource.Get())->disable(); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); } } - - -TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() { FPoseAIHandshake handshake = FPoseAIHandshake(); handshake.isMirrored = isMirrored; - handshake.rig = PoseAI_Rigs[rigIndex]; - handshake.mode = PoseAI_Modes[modeIndex]; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); handshake.syncFPS = syncFPS; handshake.cameraFPS = cameraFPS; handshake.useRootMotion = useRootMotion; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); - return MakeShared(portNum, isIPv6, handshake); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); } FReply SPoseAILiveLinkWidget::OnToggleModeClicked() @@ -302,4 +306,6 @@ void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& re TSharedPtr pin = checkBox.Pin(); if (pin) readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); -} \ No newline at end of file +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..58043dd --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h index df646d4..977dea1 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h similarity index 90% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index bf095a7..4e5bb36 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -10,7 +10,6 @@ #include "PoseAIEventDispatcher.generated.h" -#define LOCTEXT_NAMESPACE "PoseAI" DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); @@ -141,6 +140,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -268,6 +271,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -301,9 +306,24 @@ GENERATED_BODY() FPoseAIDisconnect disconnect; FPoseAIDisconnect closeSource; + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h similarity index 80% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h index ef77995..75d849b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,6 +14,7 @@ class FPoseAILiveLinkModule : public IModuleInterface virtual void StartupModule() override; virtual void ShutdownModule() override; - - +private: + }; + diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..1e00593 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,107 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; + diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..783c7d8 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,75 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; + diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h new file mode 100644 index 0000000..5a24a5b --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -0,0 +1,145 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAILiveLinkServer.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + + +struct POSEAILIVELINK_API PoseAIPortRecord { + FGuid source; + FName connectionName; + FLiveLinkSubjectKey subjectKey; +}; + +class PoseAILiveLinkSingleSourceListener; + + +/** + * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. + * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource +{ +public: + + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {} + + // custom methods + static bool GetPortGuid(int32 port, FGuid& fguid); + static bool IsValidPort(int32 port); + static FName GetConnectionName(int32 port); + static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); + static FName SubjectNameFromPort(int32 port); + + void disable(); + FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; + int32 port; + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; + + void AddSubject(); + +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h index 99a8847..1819982 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -1,4 +1,4 @@ -// Copyright PoseAI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once #include "CoreMinimal.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h similarity index 72% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h index 6121ac4..0e43854 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,21 +14,22 @@ #include "Json.h" #include "PoseAIStructs.h" #include "PoseAIUdpSocketReceiver.h" -#include "PoseAIRig.h" #include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" -#define LOCTEXT_NAMESPACE "PoseAI" class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; class FPoseAISocketSender; -class PoseAILiveLinkSingleSource; +// The networking class needs to be rewritten class POSEAILIVELINK_API PoseAILiveLinkServer { - friend PoseAILiveLinkSingleSource; public: - PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6 = false, int32 portNum = 8080); + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); ~PoseAILiveLinkServer() { CleanUp(); @@ -38,13 +39,14 @@ class POSEAILIVELINK_API PoseAILiveLinkServer static bool GetIP(FString& myIP); void CleanUp(); - void CloseTarget(const FLiveLinkSubjectName& target); void Disconnect(); - void DisconnectTarget(const FLiveLinkSubjectName& target); TSharedPtr GetSocket() const { return serverSocket; } - void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint); - void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config); + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; void SendHandshake() const; void SetHandshake(const FPoseAIHandshake& handshake); // receiver will be set on a runnable thread and set once started @@ -55,41 +57,38 @@ class POSEAILIVELINK_API PoseAILiveLinkServer const static FString fieldPrettyName; const static FString fieldVersion; const static FString fieldUUID; - static const FString fieldRigType; + const static FString fieldRigType; const static FString requiredMinVersion; - - PoseAILiveLinkSingleSource* source_ = nullptr; - + TSharedPtr listener; + TWeakPtr source_; FPoseAIHandshake handshake; FName protocolType; int32 port; + bool cleaningUp = false; // time of last connection. After timeout seconds a newer connection can takeover the port. FDateTime lastConnection; const double TIMEOUT_SECONDS = 10.0; - // dirty flag which source checks during update to see if rig static data needs updating - bool hasNewRig = false; - TSharedPtr serverSocket; - TSharedPtr rig; + //used to launch receiver without slowing main thread TSharedPtr poseAILiveLinkRunnable; + //Listens for packets TSharedPtr udpSocketReceiver; + //sends instructions to paired app TSharedPtr udpSocketSender; FPoseAIEndpoint endpoint; - - // disconnect message formatted for Pose AI mobile app FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + - bool SendString(FString& message) const; bool HasValidConnection() const; FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; @@ -101,30 +100,20 @@ class POSEAILIVELINK_API PoseAILiveLinkServer void CleanUpReceiver(); void CleanUpSender(); void CleanUpSocket(); - bool cleaningUp = false; + }; + + class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable { public: - PoseAILiveLinkReceiverRunnable(int32 port, PoseAILiveLinkServer* server) : - port(port), poseAILiveLinkServer(server) { + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); } - - virtual uint32 Run() override { - UE_LOG(LogTemp, Display, TEXT("PoseAI: Running server thread")); - FTimespan inWaitTime = FTimespan::FromMilliseconds(250); - FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); - udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, * receiverName); - udpSocketReceiver->OnDataReceived().BindRaw(poseAILiveLinkServer, &PoseAILiveLinkServer::ReceiveUDPDelegate); - udpSocketReceiver->Start(); - poseAILiveLinkServer->SetReceiver(udpSocketReceiver); - poseAILiveLinkServer = nullptr; - thread = nullptr; - return 0; - } + virtual uint32 Run() override; protected: FString myName; @@ -132,16 +121,19 @@ class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable FRunnableThread* thread = nullptr; private: PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; TSharedPtr udpSocketReceiver; }; + + // built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable { public: - FPoseAISocketSender(TSharedPtr socket, const TCHAR* threadDescription) : - socket(socket) { + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); } @@ -173,13 +165,14 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) { if (running) { + int32 sent = 0; - if (socket == nullptr) { + if (!Socket) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); return false; } - if (!socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); if (sent != Data->Num()) @@ -199,7 +192,7 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable protected: /** The network socket. */ - TSharedPtr socket; + TSharedPtr Socket; /** The thread object. */ FRunnableThread* thread = nullptr; @@ -208,3 +201,18 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool sleeping = false; }; + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h similarity index 93% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h index 61bbdc1..d662581 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -32,3 +32,5 @@ class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFa virtual FText GetSourceTooltip() const override { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } }; + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h similarity index 91% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 6d3f035..4e3fafe 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -10,7 +10,6 @@ #include "Json.h" #include "PoseAIStructs.h" -#define LOCTEXT_NAMESPACE "PoseAI" struct POSEAILIVELINK_API Remapping { @@ -64,12 +63,16 @@ class POSEAILIVELINK_API PoseAIRig bool useRootMotion; bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; + bool isCrouching = false; int32 handZoneL = 5; @@ -89,7 +92,7 @@ class POSEAILIVELINK_API PoseAIRig float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); @@ -102,6 +105,7 @@ class POSEAILIVELINK_API PoseAIRig void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +122,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; @@ -130,4 +142,5 @@ class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; protected: void Configure(); -}; \ No newline at end of file +}; + diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h similarity index 77% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index 78a665f..a09a5c6 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -8,10 +8,6 @@ #include "PoseAIStructs.generated.h" -#define LOCTEXT_NAMESPACE "PoseAI" - - - /* decoding utilities for compact representation */ float UintB64ToUint(char a, char b); uint32 UintB64ToUint(char a, char b, char c); @@ -19,64 +15,126 @@ float FixedB64pairToFloat(char a, char b); void FStringFixed12ToFloat(const FString& data, TArray& flatArray); void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + /* the handshake configures the main parameters of pose camera*/ USTRUCT(BlueprintType) -struct POSEAILIVELINK_API FPoseAIHandshake +struct POSEAILIVELINK_API FPoseAIHandshake { GENERATED_BODY() - - /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString mode = "Room"; + EPoseAiAppModes mode = EPoseAiAppModes::Room; /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString rig = "UE4"; + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; - /* the model context. Will enable new AI models as they are deployed*/ + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString context = "Default"; + bool isFaceAnimating = true; - /* the target frame rate, where phone does interpolation and smoothing for animations. Events are raw.*/ + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 syncFPS = 60; + bool isMirrored = true; - /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 cameraFPS = 60; + bool isLowerBodyRotated = false; - /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = false; + bool useRootMotion = false; - /* controls compactness of packet. 0 is verbose JSON (mainly use for debugging), 1 is fairly compact JSON (preferred as of this plugin release). We may add even more condensed formats in the future.*/ + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 packetFormat = 1; + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; - /* whether to include motion within camera frame in hips or in root*/ + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString whoami = ""; + FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString signature = ""; + FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; - FString ToString() const; + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; - bool operator!=(const FPoseAIHandshake& Other) const {return !operator==(Other);} + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; FString YesNoString(bool val) const { return val ? FString("YES") : FString("NO"); } }; + /*adjusts the sensitivity of PoseAI events*/ USTRUCT(BlueprintType) struct POSEAILIVELINK_API FPoseAIModelConfig @@ -100,7 +158,7 @@ struct POSEAILIVELINK_API FPoseAIModelConfig float jumpSensitivity = 0.5f; UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") - bool isMirrored; + bool isMirrored = true; FString ToString() const; FString YesNoString(bool val) const { @@ -277,6 +335,10 @@ struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors TArray HipScreen; UPROPERTY() TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; }; @@ -395,6 +457,21 @@ GENERATED_BODY() UPROPERTY(BlueprintReadOnly, Category="PoseAI") FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); /** if at least one foot has been stationary for a few frames */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") @@ -415,4 +492,3 @@ GENERATED_BODY() }; - diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h similarity index 88% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h index c86c3bc..9434a67 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -1,4 +1,4 @@ -// Pose AI Ltd Copyright 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. // This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) #pragma once @@ -138,6 +138,7 @@ class FPoseAIUdpSocketReceiver /** Update this socket receiver. */ void Update(const FTimespan& SocketWaitTime) { + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) { return; @@ -151,22 +152,30 @@ class FPoseAIUdpSocketReceiver TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); uint32 Size; - - while (Socket!=nullptr && Socket.IsValid() && Socket->HasPendingData(Size)) + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) { - - - int32 Read = 0; - if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), Read, *Sender)) + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) { - // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs - UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); - FString recvMessage = FString(Read, bytedata); + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); } + } - + } protected: diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h similarity index 96% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h index d74a39b..dad2201 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -55,7 +55,7 @@ class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public static bool useRootMotion; static bool isIPv6; - + static FPoseAIHandshake GetHandshake(); void UpdatePort(const FText& InText, ETextCommit::Type type); void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs rename to UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs index ad829c5..e39bf7b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -1,10 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright 2022 Pose AI Ltd. All Rights Reserved. using UnrealBuildTool; -public class PoseAILiveLink : ModuleRules +public class PoseAILiveLinkEd : ModuleRules { - public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; @@ -29,13 +29,8 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) "CoreUObject", "Engine", "InputCore", - "Projects", - "Networking", - "Sockets", - "LiveLink", - "LiveLinkInterface", - "Json", - "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", // ... add other public dependencies that you statically link with here ... } ); @@ -46,18 +41,20 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) { "Slate", "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + // ... add private dependencies that you statically link with here ... } ); - - + + DynamicallyLoadedModuleNames.AddRange( new string[] { // ... add any modules that your module loads dynamically here ... } ); - // PublicDefinitions.Add("WINDOWS_IGNORE_PACKING_MISMATCH"); - } } diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV1.4/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/PoseAILiveLink.uplugin similarity index 75% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/PoseAILiveLink.uplugin rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/PoseAILiveLink.uplugin index 149bc41..bcd7eb6 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,9 +1,9 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.33", + "VersionName": "1.41", "FriendlyName": "PoseAI LiveLink", - "Description": "Live Link plugin to stream from the Pose Camera mobile app by PoseAI", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", "CreatedBy": "Pose AI Ltd", "CreatedByURL": "www.pose-ai.com", @@ -23,6 +23,11 @@ "Win64", "Mac" ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" } ], "Plugins": [ diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Resources/Icon128.png similarity index 100% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Resources/Icon128.png rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Resources/Icon128.png diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..8d369fc --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,103 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Bone1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Bone2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Bone3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Bone4CSPos = Bone1CSPos + 0.5f * (FVector::Dist(Bone2CSPos, Bone1CSPos) + FVector::Dist(Bone3CSPos, Bone1CSPos)) * + FVector::CrossProduct(Bone3CSPos - Bone1CSPos, Bone2CSPos - Bone1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Bone1CSPos + + PoseAiIkVector.Y * Bone2CSPos + + PoseAiIkVector.Z * Bone3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Bone4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp similarity index 83% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp index ccaf3bc..662430c 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -1,4 +1,4 @@ -// Copyright 2021 Pose AI Ltd. . +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEndpoint.h" @@ -9,7 +9,11 @@ TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int FName socketType = NAME_DGram; FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); - socket->SetNonBlocking(); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + socket->SetReuseAddr(); int actualSize; socket->SetReceiveBufferSize(64 * 1024, actualSize); diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp similarity index 92% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 7189e64..6b53b2d 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -1,7 +1,7 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" void UStepCounter::Halt(bool fade) { num_ = 0; @@ -76,8 +76,8 @@ UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { - portNum = PoseAILiveLinkSingleSource::portDefault; - while (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; @@ -89,7 +89,7 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr InitializeObjects(); FLiveLinkSubjectName addedSubjectName; PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkSingleSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } @@ -111,7 +111,7 @@ bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeI InitializeObjects(); bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); if (success) - onRegistered.Broadcast(name, PoseAILiveLinkSingleSource::GetConnectionName(name)); + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); return success; } @@ -142,6 +142,23 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { } + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, myIP, portNum, isIPv6); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { + FLiveLinkSubjectName addedSubjectName; + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName); +} + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); @@ -189,7 +206,7 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam UPoseAIMovementComponent* existing_component; bool isReconnection = HasComponent(subjectName, existing_component); if (isReconnection) { - if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkSingleSource::GetConnectionName(subjectName)); + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); @@ -217,7 +234,8 @@ void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { } void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { - knownConnectionsWithTime.Add(subjectName,FDateTime::Now()); + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { UPoseAIMovementComponent* component; if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..8eb2f25 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,22 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..58ac335 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,156 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + + + +FCriticalSection critSingleSectionLocal; + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + + } + } +} diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..ff08b49 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,223 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +FCriticalSection critSingleSection; + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + critSingleSection.Lock(); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } + critSingleSection.Unlock(); +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp index 63f5013..1284488 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkRetargetRotations.h" diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp similarity index 76% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp index 5648dbe..bd297f0 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -1,21 +1,25 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkServer.h" #include "Async/Async.h" #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" -const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("0.8.24")); +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); -PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6, int32 portNum) : - source_(mySource), handshake(myHandshake), port(portNum) { +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); @@ -23,7 +27,7 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); serverSocket = BuildUdpSocket(serverName, protocolType, port); - poseAILiveLinkRunnable = MakeShared(port, this); + poseAILiveLinkRunnable = MakeShared(port, listener, this); udpSocketSender = MakeShared(serverSocket, *senderName); FString myIP; @@ -36,6 +40,10 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL } } +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + bool PoseAILiveLinkServer::GetIP(FString& myIP) { bool canBind = false; @@ -51,7 +59,6 @@ bool PoseAILiveLinkServer::GetIP(FString& myIP) { } void PoseAILiveLinkServer::CleanUp() { - source_ = nullptr; if (!cleaningUp) { cleaningUp = true; CleanUpReceiver(); @@ -68,18 +75,18 @@ void PoseAILiveLinkServer::CleanUp() { } void PoseAILiveLinkServer::CleanUpReceiver() { - if (udpSocketReceiver != nullptr && udpSocketReceiver.IsValid()) { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); udpSocketReceiver->Stop(); } - if (poseAILiveLinkRunnable != nullptr && poseAILiveLinkRunnable.IsValid()) { + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); poseAILiveLinkRunnable.Reset(); } } void PoseAILiveLinkServer::CleanUpSender() { - if (udpSocketSender != nullptr && udpSocketSender.IsValid()) { + if (udpSocketSender && udpSocketSender.IsValid()) { Disconnect(); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); udpSocketSender->Stop(); @@ -92,7 +99,7 @@ bool PoseAILiveLinkServer::HasValidConnection() const { return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; } -void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { static const FGuid GUID_Error = FGuid(); if (cleaningUp) return; @@ -105,9 +112,10 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); return; } + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); if (HasValidConnection() && !sameAsCurrent) { - if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint SendHandshake(); } @@ -115,7 +123,6 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); //consider sending rejected connection a warning message } - } else if (!HasValidConnection() && !sameAsCurrent) { // new connection InitiateConnection(jsonObject, endpointRecv); @@ -124,10 +131,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const else { if (PoseAIRig::IsFrameData(jsonObject)) { lastConnection = FDateTime::Now(); - source_->UpdatePose(rig, jsonObject); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(source_->GetSubjectName()); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } } - else if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { //is likely a repeat hello message + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message SendHandshake(); } } @@ -150,30 +160,37 @@ void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject return; } FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); - source_->SetConnectionName(connectionName); - endpoint = endpointRecv; - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - SendHandshake(); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_->GetSubjectName()); - lastConnection = FDateTime::Now(); UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } } bool PoseAILiveLinkServer::SendString(FString& message) const { - if (!endpoint.IsValid()) + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { return false; - FTCHARToUTF8 byteConvert(*message); - TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); - bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; - return udpSocketSender->Send(bytedata, endpoint); + } } + void PoseAILiveLinkServer::SendHandshake() const { FString message_string = handshake.ToString(); if (SendString(message_string)) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); - } else { //unsuccesful + } else { //unsuccessful static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; static const FGuid GUID_HandshakeFail = FGuid(); FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); @@ -182,31 +199,12 @@ void PoseAILiveLinkServer::SendHandshake() const { } void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { - bool dirty = handshake != newHandshake; - bool rigChange = handshake.rig != newHandshake.rig; handshake = newHandshake; - if (rigChange) { - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - } - if (dirty && endpoint.IsValid()) + if (endpoint.IsValid()) SendHandshake(); } -void PoseAILiveLinkServer::SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { - if (target == source_->GetSubjectName()) { - FString message_string = config.ToString(); - if (SendString(message_string)) - UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s to %s"), *message_string, *endpoint.ToString()); - } -} - -void PoseAILiveLinkServer::CloseTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - source_->RequestSourceShutdown(); -} - void PoseAILiveLinkServer::Disconnect() { if (endpoint.IsValid()) { @@ -222,10 +220,6 @@ void PoseAILiveLinkServer::Disconnect() { } } -void PoseAILiveLinkServer::DisconnectTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - Disconnect(); -} FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const { @@ -253,3 +247,17 @@ bool PoseAILiveLinkServer::CheckAppVersion(FString version) const } return true; } + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp similarity index 91% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp index 24ee263..c7b1432 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkSourceFactory.h" #include "SPoseAILiveLinkWidget.h" diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp similarity index 97% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 90e34e4..fb05e4b 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" @@ -23,30 +23,33 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), - rigType(FName(handshake.rig)), + rigType(FName(handshake.GetRigString())), useRootMotion(handshake.useRootMotion), - includeHands(!handshake.mode.Contains(TEXT("BodyOnly"))), + includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), - isDesktop(handshake.mode.Contains(TEXT("Desktop"))) { + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; - FName rigType = FName(handshake.rig); - if (rigType == FName("Mixamo")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("MetaHuman")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("DazUE")) { - rigPtr = MakeShared(name, handshake); - } - else { - rigPtr = MakeShared(name, handshake);; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; } + rigPtr->Configure(); return rigPtr; } @@ -93,7 +96,11 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink FString rigStringOut; if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } return false; } diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp similarity index 73% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index 9b5a31b..595a509 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -1,7 +1,9 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIStructs.h" +// Utility conversion functions for compact representation and from arrays to vectors + float UintB64ToUint(char a, char b) { static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; @@ -19,35 +21,51 @@ float FixedB64pairToFloat(char a, char b) { } void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { - check(data.Len() % 2 == 0); flatArray.Reserve(flatArray.Num() + data.Len() / 2); for (int i = 0; i + 1 < data.Len(); i += 2) flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); } void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { - check(flatArray.Num() % 4 == 0); quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); - for (int i = 0; i + 3< flatArray.Num(); i += 4) + for (int i = 0; i + 3 < flatArray.Num(); i += 4) quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); } void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ const TArray < TSharedPtr < FJsonValue > >* value; - if (jsonObj->TryGetArrayField(fieldName, value)){ + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ fieldVector2D.X = (*value)[0]->AsNumber(); fieldVector2D.Y = (*value)[1]->AsNumber(); } } +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { - if (value.Num() == 2) { + if (value.Num() >= 2) { fieldVector2D.X = value[0]; fieldVector2D.Y = value[1]; } } +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { if (newValue != field) changeFlag = true; @@ -56,7 +74,42 @@ void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { SetAndCheckForChange(newValue > 0.5f, field, changeFlag); } +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} FString FPoseAIHandshake::ToString() const { return FString::Printf( @@ -70,25 +123,30 @@ FString FPoseAIHandshake::ToString() const { "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"handModelVersion\": %d, " "\"packetFormat\": %d" "}}"), - *rig, - *mode, - *context, + *(GetRigString()), + *(GetModeString()), + *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, - packetFormat + GetHandModelVersion(), + static_cast(packetFormat) ); } + bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const { return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; } + + FString FPoseAIModelConfig::ToString() const { return FString::Printf( TEXT("{\"CONFIG\":{" @@ -157,24 +215,59 @@ void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { if (compactString.Len() < 12) return; - upperBodyLean.Set(FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, - FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f); - hipScreen.Set(FixedB64pairToFloat(compactString[4], compactString[5]), - FixedB64pairToFloat(compactString[6], compactString[7])); - chestScreen.Set(FixedB64pairToFloat(compactString[8], compactString[9]), - FixedB64pairToFloat(compactString[10], compactString[11])); + upperBodyLean.Set( + FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, + FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f + ); + hipScreen.Set( + FixedB64pairToFloat(compactString[4], compactString[5]), + FixedB64pairToFloat(compactString[6], compactString[7]) + ); + chestScreen.Set( + FixedB64pairToFloat(compactString[8], compactString[9]), + FixedB64pairToFloat(compactString[10], compactString[11]) + ); + if (compactString.Len() < 24) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[12], compactString[13]) * 4.0f, + FixedB64pairToFloat(compactString[14], compactString[15]) * 4.0f, + FixedB64pairToFloat(compactString[16], compactString[17]) * 4.0f + ); + handIkR.Set( + FixedB64pairToFloat(compactString[18], compactString[19]) * 4.0f, + FixedB64pairToFloat(compactString[20], compactString[21]) * 4.0f, + FixedB64pairToFloat(compactString[22], compactString[23]) * 4.0f + ); + } void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandLeft.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandLeft.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkL.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } void FPoseAILiveValues::ProcessCompactVectorsHandRight(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandRight.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandRight.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkR.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } @@ -216,6 +309,8 @@ void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ upperBodyLean *= 180.0f; ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); } @@ -224,11 +319,14 @@ void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonOb if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); + } void FPoseAILiveValues::ProcesssVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); } diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp similarity index 92% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp index ace89a0..5bbbf45 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -1,18 +1,19 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "SPoseAILiveLinkWidget.h" #include "PoseAILiveLinkSourceFactory.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" #define LOCTEXT_NAMESPACE "PoseAI" TWeakPtr SPoseAILiveLinkWidget::source = nullptr; +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; -static TArray PoseAI_Rigs = { "UE4", "MetaHuman", "Mixamo", "DazUE"}; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; -int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkSingleSource::portDefault; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; int32 SPoseAILiveLinkWidget::syncFPS = 60; int32 SPoseAILiveLinkWidget::cameraFPS = 60; int32 SPoseAILiveLinkWidget::modeIndex = 0; @@ -192,7 +193,7 @@ bool SPoseAILiveLinkWidget::IsPortValid() const return false; } - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); return false; } @@ -234,22 +235,25 @@ void SPoseAILiveLinkWidget::disableExistingSource() TSharedPtr linkSource = source.Pin(); if (linkSource.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); - ((PoseAILiveLinkSingleSource*)linkSource.Get())->disable(); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); } } - - -TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() { FPoseAIHandshake handshake = FPoseAIHandshake(); handshake.isMirrored = isMirrored; - handshake.rig = PoseAI_Rigs[rigIndex]; - handshake.mode = PoseAI_Modes[modeIndex]; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); handshake.syncFPS = syncFPS; handshake.cameraFPS = cameraFPS; handshake.useRootMotion = useRootMotion; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); - return MakeShared(portNum, isIPv6, handshake); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); } FReply SPoseAILiveLinkWidget::OnToggleModeClicked() diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h index df646d4..977dea1 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index bf095a7..fdf95f7 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -301,6 +301,15 @@ GENERATED_BODY() FPoseAIDisconnect disconnect; FPoseAIDisconnect closeSource; + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum = 8080, bool isIPv6 = false); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h similarity index 80% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h index ef77995..75d849b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,6 +14,7 @@ class FPoseAILiveLinkModule : public IModuleInterface virtual void StartupModule() override; virtual void ShutdownModule() override; - - +private: + }; + diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..854c15d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,77 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); + } + virtual FText GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; + } + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h similarity index 54% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index 77193e5..b959395 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,7 +19,7 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" -#include "PoseAIEventDispatcher.h" + #define LOCTEXT_NAMESPACE "PoseAI" @@ -30,18 +30,28 @@ struct POSEAILIVELINK_API PoseAIPortRecord { FLiveLinkSubjectKey subjectKey; }; +class PoseAILiveLinkSingleSourceListener; + + /** * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to * trigger frame events and update the LiveLink pose source information. */ -class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource { public: - static const int32 portDefault = 8080; - PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake); + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); // standard Live Link source methods virtual bool CanBeDisplayedInUI() const { return true; } @@ -53,47 +63,86 @@ class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource return LOCTEXT("SourceMachineName", "Unreal Engine");; } virtual FText GetSourceStatus() const { return status; } - virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) {} + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; virtual bool IsSourceStillValid() const override; virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} - virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid); + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; virtual bool RequestSourceShutdown(); - virtual void Update() override; - - - // method to add a source from code, instead of relying on presets. Exposed to blueprints via the PoseAI movement component. - static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + virtual void Update() override {} + // custom methods static bool GetPortGuid(int32 port, FGuid& fguid); static bool IsValidPort(int32 port); - - //Assigns a Livelink subject name from a port number. Returns FName as that class is used in LiveLinkSubjectKey consturctor - static FName SubjectNameFromPort(int32 port); - - //Looks up the "connection name" associated with port/subjetname which is the phone name @ IP Address. static FName GetConnectionName(int32 port); static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); - void SetConnectionName(FName name); - + static FName SubjectNameFromPort(int32 port); + void disable(); FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } - void UpdatePose(TSharedPtr rig, TSharedPtr jsonPose); + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); -private: - int32 port; - FGuid sourceGuid ; - FLiveLinkSubjectKey subjectKey; - TSharedPtr udpServer; + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: /* stores ports across different sources to avoid conflict from user input */ static TMap usedPorts; - ILiveLinkClient* liveLinkClient = nullptr; - ILiveLinkClient* client = nullptr; - - UPoseAIEventDispatcher* dispatcher; + TSharedPtr rig; + FPoseAIHandshake handshake; + int32 port; + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; mutable FText status; - void AddSubject(TSharedPtr rig); + void AddSubject(); + }; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h index 99a8847..1819982 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -1,4 +1,4 @@ -// Copyright PoseAI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once #include "CoreMinimal.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h index 6121ac4..908f66a 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,21 +14,23 @@ #include "Json.h" #include "PoseAIStructs.h" #include "PoseAIUdpSocketReceiver.h" -#include "PoseAIRig.h" #include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" #define LOCTEXT_NAMESPACE "PoseAI" class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; class FPoseAISocketSender; -class PoseAILiveLinkSingleSource; +// The networking class needs to be rewritten class POSEAILIVELINK_API PoseAILiveLinkServer { - friend PoseAILiveLinkSingleSource; public: - PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6 = false, int32 portNum = 8080); + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); ~PoseAILiveLinkServer() { CleanUp(); @@ -38,13 +40,14 @@ class POSEAILIVELINK_API PoseAILiveLinkServer static bool GetIP(FString& myIP); void CleanUp(); - void CloseTarget(const FLiveLinkSubjectName& target); void Disconnect(); - void DisconnectTarget(const FLiveLinkSubjectName& target); TSharedPtr GetSocket() const { return serverSocket; } - void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint); - void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config); + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; void SendHandshake() const; void SetHandshake(const FPoseAIHandshake& handshake); // receiver will be set on a runnable thread and set once started @@ -55,41 +58,38 @@ class POSEAILIVELINK_API PoseAILiveLinkServer const static FString fieldPrettyName; const static FString fieldVersion; const static FString fieldUUID; - static const FString fieldRigType; + const static FString fieldRigType; const static FString requiredMinVersion; - - PoseAILiveLinkSingleSource* source_ = nullptr; - + TSharedPtr listener; + TWeakPtr source_; FPoseAIHandshake handshake; FName protocolType; int32 port; + bool cleaningUp = false; // time of last connection. After timeout seconds a newer connection can takeover the port. FDateTime lastConnection; const double TIMEOUT_SECONDS = 10.0; - // dirty flag which source checks during update to see if rig static data needs updating - bool hasNewRig = false; - TSharedPtr serverSocket; - TSharedPtr rig; + //used to launch receiver without slowing main thread TSharedPtr poseAILiveLinkRunnable; + //Listens for packets TSharedPtr udpSocketReceiver; + //sends instructions to paired app TSharedPtr udpSocketSender; FPoseAIEndpoint endpoint; - - // disconnect message formatted for Pose AI mobile app FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + - bool SendString(FString& message) const; bool HasValidConnection() const; FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; @@ -101,30 +101,20 @@ class POSEAILIVELINK_API PoseAILiveLinkServer void CleanUpReceiver(); void CleanUpSender(); void CleanUpSocket(); - bool cleaningUp = false; + }; + + class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable { public: - PoseAILiveLinkReceiverRunnable(int32 port, PoseAILiveLinkServer* server) : - port(port), poseAILiveLinkServer(server) { + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); } - - virtual uint32 Run() override { - UE_LOG(LogTemp, Display, TEXT("PoseAI: Running server thread")); - FTimespan inWaitTime = FTimespan::FromMilliseconds(250); - FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); - udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, * receiverName); - udpSocketReceiver->OnDataReceived().BindRaw(poseAILiveLinkServer, &PoseAILiveLinkServer::ReceiveUDPDelegate); - udpSocketReceiver->Start(); - poseAILiveLinkServer->SetReceiver(udpSocketReceiver); - poseAILiveLinkServer = nullptr; - thread = nullptr; - return 0; - } + virtual uint32 Run() override; protected: FString myName; @@ -132,16 +122,19 @@ class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable FRunnableThread* thread = nullptr; private: PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; TSharedPtr udpSocketReceiver; }; + + // built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable { public: - FPoseAISocketSender(TSharedPtr socket, const TCHAR* threadDescription) : - socket(socket) { + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); } @@ -173,13 +166,14 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) { if (running) { + int32 sent = 0; - if (socket == nullptr) { + if (!Socket) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); return false; } - if (!socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); if (sent != Data->Num()) @@ -199,7 +193,7 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable protected: /** The network socket. */ - TSharedPtr socket; + TSharedPtr Socket; /** The thread object. */ FRunnableThread* thread = nullptr; @@ -208,3 +202,17 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool sleeping = false; }; + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h index 61bbdc1..6f3a137 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 6d3f035..1bfa6ba 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h similarity index 84% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index 78a665f..27fb37f 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,64 +19,107 @@ float FixedB64pairToFloat(char a, char b); void FStringFixed12ToFloat(const FString& data, TArray& flatArray); void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; + + /* the handshake configures the main parameters of pose camera*/ USTRUCT(BlueprintType) -struct POSEAILIVELINK_API FPoseAIHandshake +struct POSEAILIVELINK_API FPoseAIHandshake { GENERATED_BODY() - - /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString mode = "Room"; + EPoseAiAppModes mode = EPoseAiAppModes::Room; /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString rig = "UE4"; + EPoseAiRigPresets rig = EPoseAiRigPresets::UE4; - /* the model context. Will enable new AI models as they are deployed*/ + + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString context = "Default"; + bool isMirrored = true; - /* the target frame rate, where phone does interpolation and smoothing for animations. Events are raw.*/ + /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 syncFPS = 60; + bool useRootMotion = false; + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 cameraFPS = 60; + int32 cameraFPS = 60; - /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = false; + int32 syncFPS = 0; - /* controls compactness of packet. 0 is verbose JSON (mainly use for debugging), 1 is fairly compact JSON (preferred as of this plugin release). We may add even more condensed formats in the future.*/ + /* controls compactness of packet. */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 packetFormat = 1; - - /* whether to include motion within camera frame in hips or in root*/ + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Will enable new AI models as they are deployed*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString whoami = ""; + FString whoami = ""; /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString signature = ""; + FString signature = ""; + + bool operator==(const FPoseAIHandshake& Other) const; + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetHandModelVersion() const; FString ToString() const; - - bool operator==(const FPoseAIHandshake& Other) const; - bool operator!=(const FPoseAIHandshake& Other) const {return !operator==(Other);} FString YesNoString(bool val) const { return val ? FString("YES") : FString("NO"); } }; + /*adjusts the sensitivity of PoseAI events*/ USTRUCT(BlueprintType) struct POSEAILIVELINK_API FPoseAIModelConfig @@ -100,7 +143,7 @@ struct POSEAILIVELINK_API FPoseAIModelConfig float jumpSensitivity = 0.5f; UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") - bool isMirrored; + bool isMirrored = true; FString ToString() const; FString YesNoString(bool val) const { @@ -277,6 +320,10 @@ struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors TArray HipScreen; UPROPERTY() TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; }; @@ -395,6 +442,21 @@ GENERATED_BODY() UPROPERTY(BlueprintReadOnly, Category="PoseAI") FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); /** if at least one foot has been stationary for a few frames */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h new file mode 100644 index 0000000..e7d408c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -0,0 +1,221 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. +// This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) + +#pragma once + +#include "CoreMinimal.h" +#include "HAL/Runnable.h" +#include "HAL/RunnableThread.h" +#include "Misc/SingleThreadRunnable.h" +#include "Serialization/ArrayReader.h" +#include "Sockets.h" +#include "SocketSubsystem.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" + +#include "PoseAIEndpoint.h" +#include "IPAddress.h" + + + + + +/** + * Temporary fix for concurrency crashes. This whole class will be redesigned. + */ +typedef TSharedPtr FArrayReaderPtr; + +/** + * Delegate type for received data. + * + * The first parameter is the received data. + * The second parameter is sender's IP endpoint. + */ +DECLARE_DELEGATE_TwoParams(FPoseAIOnSocketDataReceived, const FString&, const FPoseAIEndpoint&); //Change delegate name and use our endpoint + + +/** + * Asynchronously receives data from an UDP socket. + */ +class FPoseAIUdpSocketReceiver + : public FRunnable + , private FSingleThreadRunnable +{ +public: + + /** + * Creates and initializes a new socket receiver. + * + * @param InSocket The UDP socket to receive data from. + * @param InWaitTime The amount of time to wait for the socket to be readable. + * @param InThreadName The receiver thread name (for debugging). + */ + FPoseAIUdpSocketReceiver(TSharedPtr InSocket, const FTimespan& InWaitTime, const TCHAR* InThreadName) + : Socket(InSocket) + , Stopping(false) + , Thread(nullptr) + , ThreadName(InThreadName) + , WaitTime(InWaitTime) + { + check(Socket != nullptr); + check(Socket->GetSocketType() == SOCKTYPE_Datagram); + Reader->SetNumUninitialized(MaxReadBufferSize); + SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); + } + + /** Virtual destructor. */ + virtual ~FPoseAIUdpSocketReceiver() + { + if (Thread != nullptr) + { + Thread->Kill(true); + delete Thread; + } + } + +public: + + /** Set the maximum size allocated to read off of the socket. */ + void SetMaxReadBufferSize(uint32 InMaxReadBufferSize) + { + MaxReadBufferSize = InMaxReadBufferSize; + } + + /** Start the receiver thread. */ + void Start() + { + Thread = FRunnableThread::Create(this, *ThreadName, 128 * 1024, TPri_AboveNormal, FPlatformAffinity::GetPoolThreadMask()); + } + + /** + * Returns a delegate that is executed when data has been received. + * + * This delegate must be bound before the receiver thread is started with + * the Start() method. It cannot be unbound while the thread is running. + * + * @return The delegate. + */ + FPoseAIOnSocketDataReceived& OnDataReceived() + { + check(Thread == nullptr); + return DataReceivedDelegate; + } + +public: + + //~ FRunnable interface + + virtual FSingleThreadRunnable* GetSingleThreadInterface() override + { + return this; + } + + virtual bool Init() override + { + return true; + } + + virtual uint32 Run() override + { + while (!Stopping) + { + isUpdating = true; + Update(WaitTime); + isUpdating = false; + } + + return 0; + } + + virtual void Stop() override + { + Stopping = true; + } + + virtual void Exit() override { } + +protected: + + /** Update this socket receiver. */ + void Update(const FTimespan& SocketWaitTime) + { + + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) + { + return; + } + + /************************************************* + Hwere we make changes to specify address with protocol + **********************************************************/ + if (Stopping) + return; + + TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); + uint32 Size; + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) + { + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) + { + + // UE4.2x versions + UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + FString recvMessage = FString(BytesRead, UTF8_TO_TCHAR(bytedata_utf8)); //important: keep UE MACRO inside function + // end UE4.2x + + // UE5.0 + // UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // FString recvMessage = FString(BytesRead, bytedata); + // end UE5.0 + + + DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); + } + + } + + + } + +protected: + + //~ FSingleThreadRunnable interface + + virtual void Tick() override + { + Update(FTimespan::Zero()); + } + +private: + FArrayReaderPtr Reader = MakeShared(true); + /** The network socket. */ + TSharedPtr Socket = nullptr; + + /** Pointer to the socket sub-system. */ + ISocketSubsystem* SocketSubsystem = nullptr; + + /** Flag indicating that the thread is stopping. */ + bool Stopping; + + /** The thread object. */ + FRunnableThread* Thread = nullptr; + + /** The receiver thread's name. */ + FString ThreadName; + + /** The amount of time to wait for inbound packets. */ + FTimespan WaitTime; + + /** The maximum read buffer size used to read the socket. */ + uint32 MaxReadBufferSize = 65507u; + + bool isUpdating = false; + +private: + + /** Holds the data received delegate. */ + FPoseAIOnSocketDataReceived DataReceivedDelegate; +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h similarity index 96% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h index d74a39b..dad2201 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -55,7 +55,7 @@ class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public static bool useRootMotion; static bool isIPv6; - + static FPoseAIHandshake GetHandshake(); void UpdatePort(const FText& InText, ETextCommit::Type type); void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs rename to UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs index ad829c5..e39bf7b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -1,10 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright 2022 Pose AI Ltd. All Rights Reserved. using UnrealBuildTool; -public class PoseAILiveLink : ModuleRules +public class PoseAILiveLinkEd : ModuleRules { - public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; @@ -29,13 +29,8 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) "CoreUObject", "Engine", "InputCore", - "Projects", - "Networking", - "Sockets", - "LiveLink", - "LiveLinkInterface", - "Json", - "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", // ... add other public dependencies that you statically link with here ... } ); @@ -46,18 +41,20 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) { "Slate", "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + // ... add private dependencies that you statically link with here ... } ); - - + + DynamicallyLoadedModuleNames.AddRange( new string[] { // ... add any modules that your module loads dynamically here ... } ); - // PublicDefinitions.Add("WINDOWS_IGNORE_PACKING_MISMATCH"); - } } diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.26/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/PoseAILiveLink.uplugin similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/PoseAILiveLink.uplugin rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/PoseAILiveLink.uplugin index b1808c8..3430354 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,9 +1,9 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.33", + "VersionName": "1.42", "FriendlyName": "PoseAI LiveLink", - "Description": "Live Link plugin to stream from the Pose Camera mobile app by PoseAI", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", "CreatedBy": "Pose AI Ltd", "CreatedByURL": "www.pose-ai.com", @@ -13,7 +13,6 @@ "EngineVersion": "4.27.0", "CanContainContent": false, "IsBetaVersion": false, - "IsExperimentalVersion": false, "Installed": true, "Modules": [ { @@ -21,8 +20,14 @@ "Type": "Runtime", "LoadingPhase": "Default", "WhitelistPlatforms": [ - "Win64", "Mac" + "Win64", + "Mac" ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" } ], "Plugins": [ @@ -31,5 +36,4 @@ "Enabled": true } ] - } \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Resources/Icon128.png similarity index 100% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Resources/Icon128.png rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Resources/Icon128.png diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..8d369fc --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,103 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Bone1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Bone2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Bone3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Bone4CSPos = Bone1CSPos + 0.5f * (FVector::Dist(Bone2CSPos, Bone1CSPos) + FVector::Dist(Bone3CSPos, Bone1CSPos)) * + FVector::CrossProduct(Bone3CSPos - Bone1CSPos, Bone2CSPos - Bone1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Bone1CSPos + + PoseAiIkVector.Y * Bone2CSPos + + PoseAiIkVector.Z * Bone3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Bone4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp similarity index 83% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp index ccaf3bc..662430c 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -1,4 +1,4 @@ -// Copyright 2021 Pose AI Ltd. . +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEndpoint.h" @@ -9,7 +9,11 @@ TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int FName socketType = NAME_DGram; FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); - socket->SetNonBlocking(); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + socket->SetReuseAddr(); int actualSize; socket->SetReceiveBufferSize(64 * 1024, actualSize); diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp similarity index 89% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 7189e64..61106be 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -1,7 +1,7 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" void UStepCounter::Halt(bool fade) { num_ = 0; @@ -76,8 +76,8 @@ UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { - portNum = PoseAILiveLinkSingleSource::portDefault; - while (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; @@ -89,10 +89,13 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr InitializeObjects(); FLiveLinkSubjectName addedSubjectName; PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkSingleSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -111,7 +114,7 @@ bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeI InitializeObjects(); bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); if (success) - onRegistered.Broadcast(name, PoseAILiveLinkSingleSource::GetConnectionName(name)); + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); return success; } @@ -142,9 +145,30 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { } + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -189,11 +213,13 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam UPoseAIMovementComponent* existing_component; bool isReconnection = HasComponent(subjectName, existing_component); if (isReconnection) { - if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkSingleSource::GetConnectionName(subjectName)); + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); @@ -217,7 +243,8 @@ void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { } void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { - knownConnectionsWithTime.Add(subjectName,FDateTime::Now()); + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { UPoseAIMovementComponent* component; if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..8eb2f25 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,22 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..f68e3da --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + + if (jsonPose != nullptr && jsonPose->HasTypedField("Face")) { + + + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..61c68f1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,158 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..b18a1a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,226 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp index 63f5013..1284488 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkRetargetRotations.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp similarity index 75% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp index 8ac9271..bd297f0 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -1,21 +1,25 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkServer.h" #include "Async/Async.h" #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" -const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("0.8.24")); +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); -PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6, int32 portNum) : - source_(mySource), handshake(myHandshake), port(portNum) { +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); @@ -23,7 +27,7 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); serverSocket = BuildUdpSocket(serverName, protocolType, port); - poseAILiveLinkRunnable = MakeShared(port, this); + poseAILiveLinkRunnable = MakeShared(port, listener, this); udpSocketSender = MakeShared(serverSocket, *senderName); FString myIP; @@ -36,6 +40,10 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL } } +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + bool PoseAILiveLinkServer::GetIP(FString& myIP) { bool canBind = false; @@ -51,7 +59,6 @@ bool PoseAILiveLinkServer::GetIP(FString& myIP) { } void PoseAILiveLinkServer::CleanUp() { - source_ = nullptr; if (!cleaningUp) { cleaningUp = true; CleanUpReceiver(); @@ -68,18 +75,18 @@ void PoseAILiveLinkServer::CleanUp() { } void PoseAILiveLinkServer::CleanUpReceiver() { - if (udpSocketReceiver != nullptr && udpSocketReceiver.IsValid()) { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); udpSocketReceiver->Stop(); } - if (poseAILiveLinkRunnable != nullptr && poseAILiveLinkRunnable.IsValid()) { + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); poseAILiveLinkRunnable.Reset(); } } void PoseAILiveLinkServer::CleanUpSender() { - if (udpSocketSender != nullptr && udpSocketSender.IsValid()) { + if (udpSocketSender && udpSocketSender.IsValid()) { Disconnect(); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); udpSocketSender->Stop(); @@ -92,7 +99,7 @@ bool PoseAILiveLinkServer::HasValidConnection() const { return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; } -void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { static const FGuid GUID_Error = FGuid(); if (cleaningUp) return; @@ -102,12 +109,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); - FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s"), *endpointRecv.ToString()); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); return; } + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); if (HasValidConnection() && !sameAsCurrent) { - if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint SendHandshake(); } @@ -115,7 +123,6 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); //consider sending rejected connection a warning message } - } else if (!HasValidConnection() && !sameAsCurrent) { // new connection InitiateConnection(jsonObject, endpointRecv); @@ -124,10 +131,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const else { if (PoseAIRig::IsFrameData(jsonObject)) { lastConnection = FDateTime::Now(); - source_->UpdatePose(rig, jsonObject); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(source_->GetSubjectName()); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } } - else if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { //is likely a repeat hello message + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message SendHandshake(); } } @@ -150,30 +160,37 @@ void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject return; } FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); - source_->SetConnectionName(connectionName); - endpoint = endpointRecv; - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - SendHandshake(); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_->GetSubjectName()); - lastConnection = FDateTime::Now(); UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } } bool PoseAILiveLinkServer::SendString(FString& message) const { - if (!endpoint.IsValid()) + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { return false; - FTCHARToUTF8 byteConvert(*message); - TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); - bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; - return udpSocketSender->Send(bytedata, endpoint); + } } + void PoseAILiveLinkServer::SendHandshake() const { FString message_string = handshake.ToString(); if (SendString(message_string)) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); - } else { //unsuccesful + } else { //unsuccessful static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; static const FGuid GUID_HandshakeFail = FGuid(); FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); @@ -182,31 +199,12 @@ void PoseAILiveLinkServer::SendHandshake() const { } void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { - bool dirty = handshake != newHandshake; - bool rigChange = handshake.rig != newHandshake.rig; handshake = newHandshake; - if (rigChange) { - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - } - if (dirty && endpoint.IsValid()) + if (endpoint.IsValid()) SendHandshake(); } -void PoseAILiveLinkServer::SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { - if (target == source_->GetSubjectName()) { - FString message_string = config.ToString(); - if (SendString(message_string)) - UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s to %s"), *message_string, *endpoint.ToString()); - } -} - -void PoseAILiveLinkServer::CloseTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - source_->RequestSourceShutdown(); -} - void PoseAILiveLinkServer::Disconnect() { if (endpoint.IsValid()) { @@ -222,10 +220,6 @@ void PoseAILiveLinkServer::Disconnect() { } } -void PoseAILiveLinkServer::DisconnectTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - Disconnect(); -} FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const { @@ -253,3 +247,17 @@ bool PoseAILiveLinkServer::CheckAppVersion(FString version) const } return true; } + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp similarity index 91% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp index 24ee263..c7b1432 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkSourceFactory.h" #include "SPoseAILiveLinkWidget.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp similarity index 84% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 6e85cb3..8d2646d 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" @@ -23,30 +23,36 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), - rigType(FName(handshake.rig)), - useRootMotion(handshake.useRootMotion), - includeHands(!handshake.mode.Contains(TEXT("BodyOnly"))), + rigType(FName(handshake.GetRigString())), + useRootMotion(handshake.useRootMotion), + includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), - isDesktop(handshake.mode.Contains(TEXT("Desktop"))) { + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; - FName rigType = FName(handshake.rig); - if (rigType == FName("Mixamo")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("MetaHuman")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("DazUE")) { - rigPtr = MakeShared(name, handshake); - } - else { - rigPtr = MakeShared(name, handshake);; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; } + rigPtr->Configure(); return rigPtr; } @@ -83,6 +89,7 @@ bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) { + double timestamp; jsonObject->TryGetNumberField("Timestamp", timestamp); // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. @@ -93,7 +100,11 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink FString rigStringOut; if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } return false; } @@ -109,7 +120,7 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink liveValues.rootTranslation = FVector( -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip 0.0f, //currently no body distance estimate from pose camera - 0.0f + 0.0f // could do this if game calibrates from a player starting position: liveValues.hipScreen[1] * rigHeight / liveValues.bodyHeight ); TriggerEvents(); @@ -215,22 +226,34 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { + FVector baseTranslation = FVector::ZeroVector; + // seperating into lowest body part in case we want to connect with AI later + float minTorso = 0.0f; + float minFeetZ = 0.0f; + float minKnees = 0.0f; + float minHands = 0.0f; + + if (!isDesktop) + baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() TArray componentTransform; componentTransform.Emplace(data.Transforms[0]); componentTransform.Emplace(data.Transforms[1]); - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { - baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } + // to ensure grounding in the capsule, calculates lowest Z in component space. Does not include hands + for (int32 j = 2; j < parentIndices.Num(); j++) { + componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); + if (j == rShinJoint + 1 || j == rShinJoint + 2 || j == lShinJoint + 1 || j == lShinJoint + 2) + minFeetZ = FGenericPlatformMath::Min(minFeetZ, componentTransform[j].GetTranslation().Z); + else if (j == rShinJoint || j == lShinJoint) + minKnees = FGenericPlatformMath::Min(minKnees, componentTransform[j].GetTranslation().Z); + else if (j >= numBodyJoints) + minHands = FGenericPlatformMath::Min(minHands, componentTransform[j].GetTranslation().Z); + else + minTorso = FGenericPlatformMath::Min(minTorso, componentTransform[j].GetTranslation().Z); } + + float minZ = FMath::Min(FMath::Min(minTorso, minHands), FMath::Min(minFeetZ, minKnees)); + minZ -= rootHipOffsetZ; // assigns motion either to root or to hips @@ -285,11 +308,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); - AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as psoe camera does not include the root joint + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -377,7 +404,7 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject data.Transforms.Add(transform); } - AssignCharacterMotion(data); + AssignCharacterMotion(data); CachePose(data.Transforms); hasProcessedRotations = true; @@ -422,6 +449,15 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -612,6 +648,83 @@ void PoseAIRigMixamo::Configure() rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rigHeight = 161.0f; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -705,7 +818,11 @@ void PoseAIRigMetaHuman::Configure() void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 4; + lShinJoint = 9; + lowerBodyNumOfJoints = 10; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp similarity index 71% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index aa31622..28e63ac 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -1,7 +1,9 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIStructs.h" +// Utility conversion functions for compact representation and from arrays to vectors + float UintB64ToUint(char a, char b) { static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; @@ -19,14 +21,12 @@ float FixedB64pairToFloat(char a, char b) { } void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { - check(data.Len() % 2 == 0); flatArray.Reserve(flatArray.Num() + data.Len() / 2); for (int i = 0; i + 1 < data.Len(); i += 2) flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); } void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { - check(flatArray.Num() % 4 == 0); quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); for (int i = 0; i + 3 < flatArray.Num(); i += 4) quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); @@ -35,19 +35,37 @@ void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ const TArray < TSharedPtr < FJsonValue > >* value; - if (jsonObj->TryGetArrayField(fieldName, value)){ + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ fieldVector2D.X = (*value)[0]->AsNumber(); fieldVector2D.Y = (*value)[1]->AsNumber(); } } +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { - if (value.Num() == 2) { + if (value.Num() >= 2) { fieldVector2D.X = value[0]; fieldVector2D.Y = value[1]; } } +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { if (newValue != field) changeFlag = true; @@ -56,7 +74,47 @@ void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { SetAndCheckForChange(newValue > 0.5f, field, changeFlag); } +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} FString FPoseAIHandshake::ToString() const { return FString::Printf( @@ -64,31 +122,42 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), - *rig, - *mode, - *context, + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, - packetFormat + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) ); } + bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const { return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; } + + FString FPoseAIModelConfig::ToString() const { return FString::Printf( TEXT("{\"CONFIG\":{" @@ -157,24 +226,59 @@ void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { if (compactString.Len() < 12) return; - upperBodyLean.Set(FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, - FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f); - hipScreen.Set(FixedB64pairToFloat(compactString[4], compactString[5]), - FixedB64pairToFloat(compactString[6], compactString[7])); - chestScreen.Set(FixedB64pairToFloat(compactString[8], compactString[9]), - FixedB64pairToFloat(compactString[10], compactString[11])); + upperBodyLean.Set( + FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, + FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f + ); + hipScreen.Set( + FixedB64pairToFloat(compactString[4], compactString[5]), + FixedB64pairToFloat(compactString[6], compactString[7]) + ); + chestScreen.Set( + FixedB64pairToFloat(compactString[8], compactString[9]), + FixedB64pairToFloat(compactString[10], compactString[11]) + ); + if (compactString.Len() < 24) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[12], compactString[13]) * 4.0f, + FixedB64pairToFloat(compactString[14], compactString[15]) * 4.0f, + FixedB64pairToFloat(compactString[16], compactString[17]) * 4.0f + ); + handIkR.Set( + FixedB64pairToFloat(compactString[18], compactString[19]) * 4.0f, + FixedB64pairToFloat(compactString[20], compactString[21]) * 4.0f, + FixedB64pairToFloat(compactString[22], compactString[23]) * 4.0f + ); + } void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandLeft.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandLeft.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkL.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } void FPoseAILiveValues::ProcessCompactVectorsHandRight(const FString& compactString) { if (compactString.Len() < 4) return; - pointHandRight.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); + pointHandRight.Set( + FixedB64pairToFloat(compactString[0], compactString[1]), + FixedB64pairToFloat(compactString[2], compactString[3]) + ); + if (compactString.Len() < 10) return; + fingerIkR.Set( + FixedB64pairToFloat(compactString[4], compactString[5]) * 4.0f, + FixedB64pairToFloat(compactString[6], compactString[7]) * 4.0f, + FixedB64pairToFloat(compactString[8], compactString[9]) * 4.0f + ); } @@ -206,7 +310,7 @@ void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ bodyHeight = verbose.Scalars.BodyHeight; - stableFeet = FMath::RoundToInt(verbose.Scalars.StableFoot); + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); stanceYaw = verbose.Scalars.StanceYaw * 180.0f; chestYaw = verbose.Scalars.ChestYaw * 180.0f; isCrouching = verbose.Scalars.IsCrouching > 0.5f; @@ -216,6 +320,8 @@ void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ upperBodyLean *= 180.0f; ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); } @@ -224,11 +330,14 @@ void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonOb if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); + } void FPoseAILiveValues::ProcesssVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); } diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp similarity index 92% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp index ace89a0..5bbbf45 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -1,18 +1,19 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "SPoseAILiveLinkWidget.h" #include "PoseAILiveLinkSourceFactory.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" #define LOCTEXT_NAMESPACE "PoseAI" TWeakPtr SPoseAILiveLinkWidget::source = nullptr; +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; -static TArray PoseAI_Rigs = { "UE4", "MetaHuman", "Mixamo", "DazUE"}; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; -int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkSingleSource::portDefault; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; int32 SPoseAILiveLinkWidget::syncFPS = 60; int32 SPoseAILiveLinkWidget::cameraFPS = 60; int32 SPoseAILiveLinkWidget::modeIndex = 0; @@ -192,7 +193,7 @@ bool SPoseAILiveLinkWidget::IsPortValid() const return false; } - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); return false; } @@ -234,22 +235,25 @@ void SPoseAILiveLinkWidget::disableExistingSource() TSharedPtr linkSource = source.Pin(); if (linkSource.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); - ((PoseAILiveLinkSingleSource*)linkSource.Get())->disable(); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); } } - - -TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() { FPoseAIHandshake handshake = FPoseAIHandshake(); handshake.isMirrored = isMirrored; - handshake.rig = PoseAI_Rigs[rigIndex]; - handshake.mode = PoseAI_Modes[modeIndex]; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); handshake.syncFPS = syncFPS; handshake.cameraFPS = cameraFPS; handshake.useRootMotion = useRootMotion; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); - return MakeShared(portNum, isIPv6, handshake); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); } FReply SPoseAILiveLinkWidget::OnToggleModeClicked() diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h similarity index 98% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h index df646d4..977dea1 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h similarity index 90% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index bf095a7..7403779 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -141,6 +141,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -268,6 +272,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -301,9 +307,24 @@ GENERATED_BODY() FPoseAIDisconnect disconnect; FPoseAIDisconnect closeSource; + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h similarity index 80% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h index ef77995..75d849b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,6 +14,7 @@ class FPoseAILiveLinkModule : public IModuleInterface virtual void StartupModule() override; virtual void ShutdownModule() override; - - +private: + }; + diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..7029303 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,107 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..be7e999 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,80 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); + } + virtual FText GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; + } + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h similarity index 53% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index 77193e5..f11d9a4 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,7 +19,7 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" -#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkFaceSubSource.h" #define LOCTEXT_NAMESPACE "PoseAI" @@ -30,18 +30,28 @@ struct POSEAILIVELINK_API PoseAIPortRecord { FLiveLinkSubjectKey subjectKey; }; +class PoseAILiveLinkSingleSourceListener; + + /** * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to * trigger frame events and update the LiveLink pose source information. */ -class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource { public: - static const int32 portDefault = 8080; - PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake); + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); // standard Live Link source methods virtual bool CanBeDisplayedInUI() const { return true; } @@ -53,47 +63,87 @@ class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource return LOCTEXT("SourceMachineName", "Unreal Engine");; } virtual FText GetSourceStatus() const { return status; } - virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) {} + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; virtual bool IsSourceStillValid() const override; virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} - virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid); + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; virtual bool RequestSourceShutdown(); - virtual void Update() override; - - - // method to add a source from code, instead of relying on presets. Exposed to blueprints via the PoseAI movement component. - static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + virtual void Update() override {} + // custom methods static bool GetPortGuid(int32 port, FGuid& fguid); static bool IsValidPort(int32 port); - - //Assigns a Livelink subject name from a port number. Returns FName as that class is used in LiveLinkSubjectKey consturctor - static FName SubjectNameFromPort(int32 port); - - //Looks up the "connection name" associated with port/subjetname which is the phone name @ IP Address. static FName GetConnectionName(int32 port); static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); - void SetConnectionName(FName name); - + static FName SubjectNameFromPort(int32 port); + void disable(); FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } - void UpdatePose(TSharedPtr rig, TSharedPtr jsonPose); + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - TSharedPtr udpServer; - - /* stores ports across different sources to avoid conflict from user input */ - static TMap usedPorts; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; - ILiveLinkClient* liveLinkClient = nullptr; - ILiveLinkClient* client = nullptr; - - UPoseAIEventDispatcher* dispatcher; + void AddSubject(); - mutable FText status; - - void AddSubject(TSharedPtr rig); }; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h index 99a8847..1819982 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -1,4 +1,4 @@ -// Copyright PoseAI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once #include "CoreMinimal.h" diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h index 6121ac4..908f66a 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,21 +14,23 @@ #include "Json.h" #include "PoseAIStructs.h" #include "PoseAIUdpSocketReceiver.h" -#include "PoseAIRig.h" #include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" #define LOCTEXT_NAMESPACE "PoseAI" class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; class FPoseAISocketSender; -class PoseAILiveLinkSingleSource; +// The networking class needs to be rewritten class POSEAILIVELINK_API PoseAILiveLinkServer { - friend PoseAILiveLinkSingleSource; public: - PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6 = false, int32 portNum = 8080); + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); ~PoseAILiveLinkServer() { CleanUp(); @@ -38,13 +40,14 @@ class POSEAILIVELINK_API PoseAILiveLinkServer static bool GetIP(FString& myIP); void CleanUp(); - void CloseTarget(const FLiveLinkSubjectName& target); void Disconnect(); - void DisconnectTarget(const FLiveLinkSubjectName& target); TSharedPtr GetSocket() const { return serverSocket; } - void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint); - void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config); + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; void SendHandshake() const; void SetHandshake(const FPoseAIHandshake& handshake); // receiver will be set on a runnable thread and set once started @@ -55,41 +58,38 @@ class POSEAILIVELINK_API PoseAILiveLinkServer const static FString fieldPrettyName; const static FString fieldVersion; const static FString fieldUUID; - static const FString fieldRigType; + const static FString fieldRigType; const static FString requiredMinVersion; - - PoseAILiveLinkSingleSource* source_ = nullptr; - + TSharedPtr listener; + TWeakPtr source_; FPoseAIHandshake handshake; FName protocolType; int32 port; + bool cleaningUp = false; // time of last connection. After timeout seconds a newer connection can takeover the port. FDateTime lastConnection; const double TIMEOUT_SECONDS = 10.0; - // dirty flag which source checks during update to see if rig static data needs updating - bool hasNewRig = false; - TSharedPtr serverSocket; - TSharedPtr rig; + //used to launch receiver without slowing main thread TSharedPtr poseAILiveLinkRunnable; + //Listens for packets TSharedPtr udpSocketReceiver; + //sends instructions to paired app TSharedPtr udpSocketSender; FPoseAIEndpoint endpoint; - - // disconnect message formatted for Pose AI mobile app FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + - bool SendString(FString& message) const; bool HasValidConnection() const; FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; @@ -101,30 +101,20 @@ class POSEAILIVELINK_API PoseAILiveLinkServer void CleanUpReceiver(); void CleanUpSender(); void CleanUpSocket(); - bool cleaningUp = false; + }; + + class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable { public: - PoseAILiveLinkReceiverRunnable(int32 port, PoseAILiveLinkServer* server) : - port(port), poseAILiveLinkServer(server) { + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); } - - virtual uint32 Run() override { - UE_LOG(LogTemp, Display, TEXT("PoseAI: Running server thread")); - FTimespan inWaitTime = FTimespan::FromMilliseconds(250); - FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); - udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, * receiverName); - udpSocketReceiver->OnDataReceived().BindRaw(poseAILiveLinkServer, &PoseAILiveLinkServer::ReceiveUDPDelegate); - udpSocketReceiver->Start(); - poseAILiveLinkServer->SetReceiver(udpSocketReceiver); - poseAILiveLinkServer = nullptr; - thread = nullptr; - return 0; - } + virtual uint32 Run() override; protected: FString myName; @@ -132,16 +122,19 @@ class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable FRunnableThread* thread = nullptr; private: PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; TSharedPtr udpSocketReceiver; }; + + // built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable { public: - FPoseAISocketSender(TSharedPtr socket, const TCHAR* threadDescription) : - socket(socket) { + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); } @@ -173,13 +166,14 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) { if (running) { + int32 sent = 0; - if (socket == nullptr) { + if (!Socket) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); return false; } - if (!socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); if (sent != Data->Num()) @@ -199,7 +193,7 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable protected: /** The network socket. */ - TSharedPtr socket; + TSharedPtr Socket; /** The thread object. */ FRunnableThread* thread = nullptr; @@ -208,3 +202,17 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool sleeping = false; }; + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h index 61bbdc1..6f3a137 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h similarity index 91% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 6d3f035..5aa5894 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -64,12 +64,16 @@ class POSEAILIVELINK_API PoseAIRig bool useRootMotion; bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; + bool isCrouching = false; int32 handZoneL = 5; @@ -89,7 +93,7 @@ class POSEAILIVELINK_API PoseAIRig float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); @@ -102,6 +106,7 @@ class POSEAILIVELINK_API PoseAIRig void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +123,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h similarity index 77% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index 78a665f..2f7ee9e 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,64 +19,126 @@ float FixedB64pairToFloat(char a, char b); void FStringFixed12ToFloat(const FString& data, TArray& flatArray); void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + /* the handshake configures the main parameters of pose camera*/ USTRUCT(BlueprintType) -struct POSEAILIVELINK_API FPoseAIHandshake +struct POSEAILIVELINK_API FPoseAIHandshake { GENERATED_BODY() - - /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString mode = "Room"; + EPoseAiAppModes mode = EPoseAiAppModes::Room; /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString rig = "UE4"; + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; - /* the model context. Will enable new AI models as they are deployed*/ + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString context = "Default"; + bool isFaceAnimating = true; - /* the target frame rate, where phone does interpolation and smoothing for animations. Events are raw.*/ + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 syncFPS = 60; + bool isMirrored = true; - /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 cameraFPS = 60; + bool isLowerBodyRotated = false; - /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + /* whether to include motion within camera frame in hips or in root*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = false; + bool useRootMotion = false; - /* controls compactness of packet. 0 is verbose JSON (mainly use for debugging), 1 is fairly compact JSON (preferred as of this plugin release). We may add even more condensed formats in the future.*/ + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 packetFormat = 1; + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; - /* whether to include motion within camera frame in hips or in root*/ + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString whoami = ""; + FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString signature = ""; + FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; - FString ToString() const; + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; - bool operator!=(const FPoseAIHandshake& Other) const {return !operator==(Other);} + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; FString YesNoString(bool val) const { return val ? FString("YES") : FString("NO"); } }; + /*adjusts the sensitivity of PoseAI events*/ USTRUCT(BlueprintType) struct POSEAILIVELINK_API FPoseAIModelConfig @@ -100,7 +162,7 @@ struct POSEAILIVELINK_API FPoseAIModelConfig float jumpSensitivity = 0.5f; UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") - bool isMirrored; + bool isMirrored = true; FString ToString() const; FString YesNoString(bool val) const { @@ -277,6 +339,10 @@ struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors TArray HipScreen; UPROPERTY() TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; }; @@ -395,6 +461,21 @@ GENERATED_BODY() UPROPERTY(BlueprintReadOnly, Category="PoseAI") FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); /** if at least one foot has been stationary for a few frames */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h new file mode 100644 index 0000000..e7d408c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -0,0 +1,221 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. +// This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) + +#pragma once + +#include "CoreMinimal.h" +#include "HAL/Runnable.h" +#include "HAL/RunnableThread.h" +#include "Misc/SingleThreadRunnable.h" +#include "Serialization/ArrayReader.h" +#include "Sockets.h" +#include "SocketSubsystem.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" + +#include "PoseAIEndpoint.h" +#include "IPAddress.h" + + + + + +/** + * Temporary fix for concurrency crashes. This whole class will be redesigned. + */ +typedef TSharedPtr FArrayReaderPtr; + +/** + * Delegate type for received data. + * + * The first parameter is the received data. + * The second parameter is sender's IP endpoint. + */ +DECLARE_DELEGATE_TwoParams(FPoseAIOnSocketDataReceived, const FString&, const FPoseAIEndpoint&); //Change delegate name and use our endpoint + + +/** + * Asynchronously receives data from an UDP socket. + */ +class FPoseAIUdpSocketReceiver + : public FRunnable + , private FSingleThreadRunnable +{ +public: + + /** + * Creates and initializes a new socket receiver. + * + * @param InSocket The UDP socket to receive data from. + * @param InWaitTime The amount of time to wait for the socket to be readable. + * @param InThreadName The receiver thread name (for debugging). + */ + FPoseAIUdpSocketReceiver(TSharedPtr InSocket, const FTimespan& InWaitTime, const TCHAR* InThreadName) + : Socket(InSocket) + , Stopping(false) + , Thread(nullptr) + , ThreadName(InThreadName) + , WaitTime(InWaitTime) + { + check(Socket != nullptr); + check(Socket->GetSocketType() == SOCKTYPE_Datagram); + Reader->SetNumUninitialized(MaxReadBufferSize); + SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); + } + + /** Virtual destructor. */ + virtual ~FPoseAIUdpSocketReceiver() + { + if (Thread != nullptr) + { + Thread->Kill(true); + delete Thread; + } + } + +public: + + /** Set the maximum size allocated to read off of the socket. */ + void SetMaxReadBufferSize(uint32 InMaxReadBufferSize) + { + MaxReadBufferSize = InMaxReadBufferSize; + } + + /** Start the receiver thread. */ + void Start() + { + Thread = FRunnableThread::Create(this, *ThreadName, 128 * 1024, TPri_AboveNormal, FPlatformAffinity::GetPoolThreadMask()); + } + + /** + * Returns a delegate that is executed when data has been received. + * + * This delegate must be bound before the receiver thread is started with + * the Start() method. It cannot be unbound while the thread is running. + * + * @return The delegate. + */ + FPoseAIOnSocketDataReceived& OnDataReceived() + { + check(Thread == nullptr); + return DataReceivedDelegate; + } + +public: + + //~ FRunnable interface + + virtual FSingleThreadRunnable* GetSingleThreadInterface() override + { + return this; + } + + virtual bool Init() override + { + return true; + } + + virtual uint32 Run() override + { + while (!Stopping) + { + isUpdating = true; + Update(WaitTime); + isUpdating = false; + } + + return 0; + } + + virtual void Stop() override + { + Stopping = true; + } + + virtual void Exit() override { } + +protected: + + /** Update this socket receiver. */ + void Update(const FTimespan& SocketWaitTime) + { + + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) + { + return; + } + + /************************************************* + Hwere we make changes to specify address with protocol + **********************************************************/ + if (Stopping) + return; + + TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); + uint32 Size; + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) + { + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) + { + + // UE4.2x versions + UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + FString recvMessage = FString(BytesRead, UTF8_TO_TCHAR(bytedata_utf8)); //important: keep UE MACRO inside function + // end UE4.2x + + // UE5.0 + // UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // FString recvMessage = FString(BytesRead, bytedata); + // end UE5.0 + + + DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); + } + + } + + + } + +protected: + + //~ FSingleThreadRunnable interface + + virtual void Tick() override + { + Update(FTimespan::Zero()); + } + +private: + FArrayReaderPtr Reader = MakeShared(true); + /** The network socket. */ + TSharedPtr Socket = nullptr; + + /** Pointer to the socket sub-system. */ + ISocketSubsystem* SocketSubsystem = nullptr; + + /** Flag indicating that the thread is stopping. */ + bool Stopping; + + /** The thread object. */ + FRunnableThread* Thread = nullptr; + + /** The receiver thread's name. */ + FString ThreadName; + + /** The amount of time to wait for inbound packets. */ + FTimespan WaitTime; + + /** The maximum read buffer size used to read the socket. */ + uint32 MaxReadBufferSize = 65507u; + + bool isUpdating = false; + +private: + + /** Holds the data received delegate. */ + FPoseAIOnSocketDataReceived DataReceivedDelegate; +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h similarity index 96% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h index d74a39b..dad2201 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -55,7 +55,7 @@ class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public static bool useRootMotion; static bool isIPv6; - + static FPoseAIHandshake GetHandshake(); void UpdatePort(const FText& InText, ETextCommit::Type type); void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs similarity index 73% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs rename to UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs index ad829c5..e39bf7b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -1,10 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright 2022 Pose AI Ltd. All Rights Reserved. using UnrealBuildTool; -public class PoseAILiveLink : ModuleRules +public class PoseAILiveLinkEd : ModuleRules { - public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; @@ -29,13 +29,8 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) "CoreUObject", "Engine", "InputCore", - "Projects", - "Networking", - "Sockets", - "LiveLink", - "LiveLinkInterface", - "Json", - "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", // ... add other public dependencies that you statically link with here ... } ); @@ -46,18 +41,20 @@ public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) { "Slate", "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + // ... add private dependencies that you statically link with here ... } ); - - + + DynamicallyLoadedModuleNames.AddRange( new string[] { // ... add any modules that your module loads dynamically here ... } ); - // PublicDefinitions.Add("WINDOWS_IGNORE_PACKING_MISMATCH"); - } } diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/4.27/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/PoseAILiveLink.uplugin similarity index 75% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/PoseAILiveLink.uplugin rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/PoseAILiveLink.uplugin index 7826a9f..30b6c67 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/PoseAILiveLink.uplugin +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/PoseAILiveLink.uplugin @@ -1,9 +1,9 @@ { "FileVersion": 3, "Version": 1, - "VersionName": "1.36", + "VersionName": "3.0.2", "FriendlyName": "PoseAI LiveLink", - "Description": "Live Link plugin to stream from the Pose Camera mobile app by PoseAI", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", "Category": "Animation", "CreatedBy": "Pose AI Ltd", "CreatedByURL": "www.pose-ai.com", @@ -23,6 +23,11 @@ "Win64", "Mac" ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" } ], "Plugins": [ diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Resources/Icon128.png similarity index 100% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Resources/Icon128.png rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Resources/Icon128.png diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..6042e8f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,107 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp similarity index 79% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp index ccaf3bc..6a2d82f 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -1,7 +1,8 @@ -// Copyright 2021 Pose AI Ltd. . +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEndpoint.h" +#define LOCTEXT_NAMESPACE "PoseAI" ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; @@ -9,7 +10,11 @@ TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int FName socketType = NAME_DGram; FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); - socket->SetNonBlocking(); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + socket->SetReuseAddr(); int actualSize; socket->SetReceiveBufferSize(64 * 1024, actualSize); @@ -34,4 +39,5 @@ void FPoseAIEndpoint::Initialize() CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); } +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp similarity index 79% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp index 7189e64..440a2df 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -1,7 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + void UStepCounter::Halt(bool fade) { num_ = 0; @@ -76,8 +79,8 @@ UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { - portNum = PoseAILiveLinkSingleSource::portDefault; - while (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { portNum++; if (portNum > 49151) return false; @@ -89,10 +92,13 @@ bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FStr InitializeObjects(); FLiveLinkSubjectName addedSubjectName; PoseAILiveLinkServer::GetIP(myIP); - return PoseAILiveLinkSingleSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); } +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} void UPoseAIMovementComponent::InitializeObjects() { footsteps = NewObject(); @@ -111,7 +117,7 @@ bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeI InitializeObjects(); bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); if (success) - onRegistered.Broadcast(name, PoseAILiveLinkSingleSource::GetConnectionName(name)); + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); return success; } @@ -141,10 +147,78 @@ void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { UPoseAIEventDispatcher::GetDispatcher()->SetHandshake(handshake); } +void UPoseAIMovementComponent::ScaleMotion(float RigHeight, FVector Scale) { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->rigHeight = RigHeight; + lockedRig->liveValues.scaleMotion = Scale; + } +} + +void UPoseAIMovementComponent::SetLiveCameraRotation(float pitch, float yaw, float roll){ + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + // order changed due to rotated root bone in UE + lockedRig->liveValues.cameraRotation = FRotator(roll, yaw, pitch); + } +} + + +void UPoseAIMovementComponent::UseCurrentPoseToOrientCamera() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig==nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + float pitch = -lockedRig->liveValues.upperBodyLean.Y; + float yaw = -lockedRig->liveValues.chestYaw; + lockedRig->liveValues.cameraRotation = FRotator(0.0f, yaw, pitch); + } +} +void UPoseAIMovementComponent::UseCurrentPoseAsBaseTranslation() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.rootOffset = lockedRig->liveValues.rootTranslation; + } +} + +void UPoseAIMovementComponent::ZeroMotion() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.scaleMotion = FVector3d::Zero(); + } +} + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); - + LastMovementComponent = component; UPoseAIMovementComponent* existing_component; if (HasComponent(name, existing_component)) { if (!siezeIfTaken) return false; @@ -189,11 +263,13 @@ void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectNam UPoseAIMovementComponent* existing_component; bool isReconnection = HasComponent(subjectName, existing_component); if (isReconnection) { - if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkSingleSource::GetConnectionName(subjectName)); + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); } else if (!componentQueue.IsEmpty()) { UPoseAIMovementComponent* component; componentQueue.Dequeue(component); - if (component != nullptr && IsValid(component)) component->RegisterAs(subjectName, true); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } } subjectConnected.Broadcast(subjectName, isReconnection); }); @@ -211,13 +287,16 @@ void UPoseAIEventDispatcher::BroadcastCloseSource(const FLiveLinkSubjectName& su closeSource.Broadcast(subjectName); } - +void UPoseAIEventDispatcher::BroadcastResetLivePosition() { + resetLivePositionEvent.Broadcast(); +} void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { handshakeUpdate.Broadcast(handshake); } void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { - knownConnectionsWithTime.Add(subjectName,FDateTime::Now()); + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { UPoseAIMovementComponent* component; if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); @@ -384,4 +463,6 @@ bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPos return false; component = componentsByName[name]; return component != nullptr && IsValid(component); -} \ No newline at end of file +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..fbc882c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..004c575 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + if (jsonPose != nullptr && jsonPose->HasField("Face")) { + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..31b50b0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,169 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..f3dd836 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp similarity index 62% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp index 63f5013..7c6b37e 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkRetargetRotations.h" @@ -46,6 +46,10 @@ void UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled(UBlueprint* Targ void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) { + auto rootTransform = InFrameData->Transforms[0]; + auto rootTranslation = rootTransform.GetTranslation() * scaleTranslation; + auto rootZ = FVector3d(0.0f, 0.0f, rootTranslation.Z); + auto rootXY = FVector3d(rootTranslation.X, rootTranslation.Y, 0.0f); for (int32 i = 0; i < InSkeletonData->BoneNames.Num(); i++) { FName boneName = InSkeletonData->BoneNames[i]; @@ -54,19 +58,23 @@ void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTim if (meshIndex != INDEX_NONE) { FCompactPoseBoneIndex boneIndex = OutPose.GetBoneContainer().MakeCompactPoseIndex(FMeshPoseBoneIndex(meshIndex)); - if (boneIndex != INDEX_NONE) + if (i == 0) { - // root and hip we care about location so use livelink location and scale by property which can be edited in blueprints (to compensate for different skeleton sizes) - if (i < 2) { - jointTransform.SetLocation(jointTransform.GetLocation() * scaleTranslation); - jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); - } - // for the rest we use ref pose to set bone location, to preserve skeleton dimensions - else - { - jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); - jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); - } + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootXY); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + + else if (i == 1) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootZ); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + else if (boneIndex != INDEX_NONE) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); OutPose[boneIndex] = jointTransform; } } diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp similarity index 74% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp index 8ac9271..ff8da5c 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -1,21 +1,26 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkServer.h" #include "Async/Async.h" #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" +#define LOCTEXT_NAMESPACE "PoseAI" -const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("0.8.24")); +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); -PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6, int32 portNum) : - source_(mySource), handshake(myHandshake), port(portNum) { +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); @@ -23,7 +28,7 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); serverSocket = BuildUdpSocket(serverName, protocolType, port); - poseAILiveLinkRunnable = MakeShared(port, this); + poseAILiveLinkRunnable = MakeShared(port, listener, this); udpSocketSender = MakeShared(serverSocket, *senderName); FString myIP; @@ -36,6 +41,10 @@ PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAIL } } +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + bool PoseAILiveLinkServer::GetIP(FString& myIP) { bool canBind = false; @@ -51,7 +60,6 @@ bool PoseAILiveLinkServer::GetIP(FString& myIP) { } void PoseAILiveLinkServer::CleanUp() { - source_ = nullptr; if (!cleaningUp) { cleaningUp = true; CleanUpReceiver(); @@ -68,18 +76,18 @@ void PoseAILiveLinkServer::CleanUp() { } void PoseAILiveLinkServer::CleanUpReceiver() { - if (udpSocketReceiver != nullptr && udpSocketReceiver.IsValid()) { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); udpSocketReceiver->Stop(); } - if (poseAILiveLinkRunnable != nullptr && poseAILiveLinkRunnable.IsValid()) { + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); poseAILiveLinkRunnable.Reset(); } } void PoseAILiveLinkServer::CleanUpSender() { - if (udpSocketSender != nullptr && udpSocketSender.IsValid()) { + if (udpSocketSender && udpSocketSender.IsValid()) { Disconnect(); UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); udpSocketSender->Stop(); @@ -92,7 +100,7 @@ bool PoseAILiveLinkServer::HasValidConnection() const { return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; } -void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { static const FGuid GUID_Error = FGuid(); if (cleaningUp) return; @@ -102,12 +110,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); - FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s"), *endpointRecv.ToString()); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); return; } + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); if (HasValidConnection() && !sameAsCurrent) { - if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint SendHandshake(); } @@ -115,7 +124,6 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); //consider sending rejected connection a warning message } - } else if (!HasValidConnection() && !sameAsCurrent) { // new connection InitiateConnection(jsonObject, endpointRecv); @@ -124,10 +132,13 @@ void PoseAILiveLinkServer::ReceiveUDPDelegate(const FString& recvMessage, const else { if (PoseAIRig::IsFrameData(jsonObject)) { lastConnection = FDateTime::Now(); - source_->UpdatePose(rig, jsonObject); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(source_->GetSubjectName()); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } } - else if (ExtractConnectionName(jsonObject, endpointRecv) == source_->GetConnectionName(port)) { //is likely a repeat hello message + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message SendHandshake(); } } @@ -150,30 +161,37 @@ void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject return; } FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); - source_->SetConnectionName(connectionName); - endpoint = endpointRecv; - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - SendHandshake(); - UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_->GetSubjectName()); - lastConnection = FDateTime::Now(); UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } } bool PoseAILiveLinkServer::SendString(FString& message) const { - if (!endpoint.IsValid()) + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { return false; - FTCHARToUTF8 byteConvert(*message); - TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); - bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; - return udpSocketSender->Send(bytedata, endpoint); + } } + void PoseAILiveLinkServer::SendHandshake() const { FString message_string = handshake.ToString(); if (SendString(message_string)) { UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); - } else { //unsuccesful + } else { //unsuccessful static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; static const FGuid GUID_HandshakeFail = FGuid(); FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); @@ -182,31 +200,12 @@ void PoseAILiveLinkServer::SendHandshake() const { } void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { - bool dirty = handshake != newHandshake; - bool rigChange = handshake.rig != newHandshake.rig; handshake = newHandshake; - if (rigChange) { - rig = PoseAIRig::PoseAIRigFactory(source_->GetSubjectName(), handshake); - hasNewRig = true; - } - if (dirty && endpoint.IsValid()) + if (endpoint.IsValid()) SendHandshake(); } -void PoseAILiveLinkServer::SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { - if (target == source_->GetSubjectName()) { - FString message_string = config.ToString(); - if (SendString(message_string)) - UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s to %s"), *message_string, *endpoint.ToString()); - } -} - -void PoseAILiveLinkServer::CloseTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - source_->RequestSourceShutdown(); -} - void PoseAILiveLinkServer::Disconnect() { if (endpoint.IsValid()) { @@ -222,10 +221,6 @@ void PoseAILiveLinkServer::Disconnect() { } } -void PoseAILiveLinkServer::DisconnectTarget(const FLiveLinkSubjectName& target) { - if (target == source_->GetSubjectName()) - Disconnect(); -} FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const { @@ -241,8 +236,13 @@ bool PoseAILiveLinkServer::CheckAppVersion(FString version) const UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: App version %s vs required version %s."), *version, *requiredMinVersion); TArray appArray; version.ParseIntoArray(appArray, TEXT("."), false); + if (appArray.Num() < 3) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: App version %s format error"), *version); + } + TArray requiredArray; requiredMinVersion.ParseIntoArray(requiredArray, TEXT("."), false); + for (int32 i = 0; i < 3; i++) { int32 app = FCString::Atoi(*appArray[i]); int32 req = FCString::Atoi(*requiredArray[i]); @@ -253,3 +253,19 @@ bool PoseAILiveLinkServer::CheckAppVersion(FString version) const } return true; } + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp similarity index 58% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp index 24ee263..3eddf9f 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -1,8 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAILiveLinkSourceFactory.h" #include "SPoseAILiveLinkWidget.h" +#define LOCTEXT_NAMESPACE "PoseAI" + TSharedPtr UPoseAILiveLinkSourceFactory::BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const { @@ -16,3 +18,8 @@ TSharedPtr< ILiveLinkSource > UPoseAILiveLinkSourceFactory::CreateSource(const F { return SPoseAILiveLinkWidget::CreateSource(ConnectionString); } + +FText UPoseAILiveLinkSourceFactory::GetSourceDisplayName() const { return LOCTEXT("Pose AI App", "Pose AI App"); } +FText UPoseAILiveLinkSourceFactory::GetSourceTooltip() const { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp similarity index 84% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 6e85cb3..ef6cf16 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -1,8 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" +#define LOCTEXT_NAMESPACE "PoseAI" + const FString PoseAIRig::fieldBody = FString(TEXT("Body")); const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); @@ -12,7 +14,7 @@ const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); - +TMap> PoseAIRig::RigMap = {}; bool isDifferentAndSet(int32 newValue, int32& storedValue) { bool isDifferent = newValue != storedValue; @@ -23,34 +25,45 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), - rigType(FName(handshake.rig)), - useRootMotion(handshake.useRootMotion), - includeHands(!handshake.mode.Contains(TEXT("BodyOnly"))), + rigType(FName(handshake.GetRigString())), + includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), - isDesktop(handshake.mode.Contains(TEXT("Desktop"))) { + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; - FName rigType = FName(handshake.rig); - if (rigType == FName("Mixamo")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("MetaHuman")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("DazUE")) { - rigPtr = MakeShared(name, handshake); - } - else { - rigPtr = MakeShared(name, handshake);; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; } + rigPtr->Configure(); + RigMap.Add(name, rigPtr); return rigPtr; } +TWeakPtr PoseAIRig::GetRigFromSubjectName(const FLiveLinkSubjectName& name) { + return RigMap.Contains(name)? RigMap[name] : nullptr; +} + + FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ FLiveLinkStaticDataStruct staticData; staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); @@ -83,6 +96,7 @@ bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) { + double timestamp; jsonObject->TryGetNumberField("Timestamp", timestamp); // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. @@ -93,7 +107,11 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink FString rigStringOut; if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } return false; } @@ -105,13 +123,6 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink else ProcessVerboseSupplementaryData(jsonObject, data); - if (visibilityFlags.isTorso && liveValues.bodyHeight > 0.0f) - liveValues.rootTranslation = FVector( - -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip - 0.0f, //currently no body distance estimate from pose camera - 0.0f - ); - TriggerEvents(); data.WorldTime = FPlatformTime::Seconds(); @@ -203,45 +214,19 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; if (objHandLeft != nullptr && objHandLeft.IsValid()) { - FString VecA = (objHandLeft->HasTypedField("VecA")) ? objHandLeft->GetStringField("VecA") : ""; - liveValues.ProcessCompactVectorsHandLeft(VecA); + liveValues.ProcessCompactVectorsHandLeft(objHandLeft); } objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; if (objHandRight != nullptr && objHandRight.IsValid()) { - FString VecA = (objHandRight->HasTypedField("VecA")) ? objHandRight->GetStringField("VecA") : ""; - liveValues.ProcessCompactVectorsHandRight(VecA); + liveValues.ProcessCompactVectorsHandRight(objHandRight); } } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { - - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() - TArray componentTransform; - componentTransform.Emplace(data.Transforms[0]); - componentTransform.Emplace(data.Transforms[1]); - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { - baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } - } - minZ -= rootHipOffsetZ; - - // assigns motion either to root or to hips - if (useRootMotion) { - //hip to low point Z distance assigned to hips, rest of movement assigned to root - data.Transforms[1].SetTranslation(FVector(0.0f, 0.0f, -minZ)); - data.Transforms[0].SetTranslation(baseTranslation); - } - else { - baseTranslation.Z -= minZ; - data.Transforms[1].SetTranslation(baseTranslation); + if (!isDesktop) { + FVector playerMotion = liveValues.cameraRotation.RotateVector(liveValues.rootTranslation - liveValues.rootOffset) * rigHeight * liveValues.scaleMotion; + data.Transforms[0].SetTranslation(playerMotion); } } @@ -285,11 +270,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); - AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as psoe camera does not include the root joint + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -376,7 +365,6 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations data.Transforms.Add(transform); } - AssignCharacterMotion(data); CachePose(data.Transforms); @@ -407,21 +395,34 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; } + liveValues.ProcessVerboseBody(verbose); + if (objHandLeft != nullptr && objHandLeft.IsValid()) { vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; - + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + if (objHandLeft->HasTypedField("Open")) + liveValues.opennessLeftHand = objHandLeft->GetNumberField("Open"); } if (objHandRight != nullptr && objHandRight.IsValid()) { vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandRight(vecHandRight); + if (objHandRight->HasTypedField("Open")) + liveValues.opennessRightHand = objHandRight->GetNumberField("Open"); } - liveValues.ProcessVerboseBody(verbose); - liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); - liveValues.ProcesssVerboseVectorsHandRight(vecHandRight); liveValues.jumpHeight = verbose.Events.Jump.Magnitude; visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -453,7 +454,6 @@ void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& com } void PoseAIRig::CachePose(const TArray& transforms) { - //cachedPose.Empty(); cachedPose = TArray(transforms); } @@ -531,7 +531,6 @@ void PoseAIRigUE4::Configure() } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 170.0f; rig = MakeStaticData(); } @@ -608,10 +607,85 @@ void PoseAIRigMixamo::Configure() AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 161.0f; rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -699,13 +773,16 @@ void PoseAIRigMetaHuman::Configure() AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 168.0f; rig = MakeStaticData(); } void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 5; + lShinJoint = 10; + lowerBodyNumOfJoints = 11; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); @@ -793,9 +870,8 @@ void PoseAIRigDazUE::Configure() AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 168.0f; rig = MakeStaticData(); } - +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp similarity index 58% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp index aa31622..eec8f4a 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -1,7 +1,11 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIStructs.h" +#define LOCTEXT_NAMESPACE "PoseAI" + +// Utility conversion functions for compact representation and from arrays to vectors + float UintB64ToUint(char a, char b) { static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; @@ -19,14 +23,12 @@ float FixedB64pairToFloat(char a, char b) { } void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { - check(data.Len() % 2 == 0); flatArray.Reserve(flatArray.Num() + data.Len() / 2); for (int i = 0; i + 1 < data.Len(); i += 2) flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); } void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { - check(flatArray.Num() % 4 == 0); quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); for (int i = 0; i + 3 < flatArray.Num(); i += 4) quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); @@ -35,19 +37,37 @@ void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ const TArray < TSharedPtr < FJsonValue > >* value; - if (jsonObj->TryGetArrayField(fieldName, value)){ + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ fieldVector2D.X = (*value)[0]->AsNumber(); fieldVector2D.Y = (*value)[1]->AsNumber(); } } +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { - if (value.Num() == 2) { + if (value.Num() >= 2) { fieldVector2D.X = value[0]; fieldVector2D.Y = value[1]; } } +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { if (newValue != field) changeFlag = true; @@ -56,7 +76,47 @@ void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { SetAndCheckForChange(newValue > 0.5f, field, changeFlag); } +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} FString FPoseAIHandshake::ToString() const { return FString::Printf( @@ -64,31 +124,42 @@ FString FPoseAIHandshake::ToString() const { "\"name\":\"Unreal LiveLink\"," "\"rig\":\"%s\", " "\"mode\":\"%s\", " + "\"face\":\"%s\", " "\"context\":\"%s\", " "\"whoami\":\"%s\", " "\"signature\":\"%s\", " "\"mirror\":\"%s\", " "\"syncFPS\": %d, " "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " "\"packetFormat\": %d" "}}"), - *rig, - *mode, - *context, + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), *whoami, *signature, *(YesNoString(isMirrored)), syncFPS, cameraFPS, - packetFormat + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) ); } + bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const { return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; } + + FString FPoseAIModelConfig::ToString() const { return FString::Printf( TEXT("{\"CONFIG\":{" @@ -142,43 +213,113 @@ void FPoseAIVisibilityFlags::ProcessCompact(const FString& visString) { SetAndCheckForChange(visString[2] != '0', isRightLeg, hasChanged); SetAndCheckForChange(visString[3] != '0', isLeftArm, hasChanged); SetAndCheckForChange(visString[4] != '0', isRightArm, hasChanged); + if (visString.Len() > 5) + SetAndCheckForChange(visString[5] != '0', isFace, hasChanged); } void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) { + int32 idx = 0; if(compactString.Len() < 14) return; - bodyHeight = FixedB64pairToFloat(compactString[0], compactString[1]) + 1.0f; - chestYaw = FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f; - stanceYaw = FixedB64pairToFloat(compactString[4], compactString[5]) * 180.0f; - stableFeet = UintB64ToUint(compactString[6], compactString[7]); - handZoneLeft = UintB64ToUint(compactString[8], compactString[9]); - handZoneRight = UintB64ToUint(compactString[10], compactString[11]); - isCrouching = UintB64ToUint(compactString[12], compactString[13]) > 0; + bodyHeight = FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) + 1.0f; + chestYaw = FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f; + stanceYaw = FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 180.0f; + stableFeet = UintB64ToUint(compactString[idx + 6], compactString[idx + 7]); + handZoneLeft = UintB64ToUint(compactString[idx + 8], compactString[idx + 9]); + handZoneRight = UintB64ToUint(compactString[idx + 10], compactString[idx + 11]); + isCrouching = UintB64ToUint(compactString[idx + 12], compactString[idx + 13]) > 0; } void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { + //tbd - this could be simplified if we don't need to keep supported older versions of the api + int32 idx = 0; if (compactString.Len() < 12) return; - upperBodyLean.Set(FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f, - FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f); - hipScreen.Set(FixedB64pairToFloat(compactString[4], compactString[5]), - FixedB64pairToFloat(compactString[6], compactString[7])); - chestScreen.Set(FixedB64pairToFloat(compactString[8], compactString[9]), - FixedB64pairToFloat(compactString[10], compactString[11])); + upperBodyLean.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 180.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f + ); + idx += 4; + hipScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx+1]), + FixedB64pairToFloat(compactString[idx+2], compactString[idx+3]) + ); + idx += 4; + chestScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]), + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) + ); + idx += 4; + if (compactString.Len() < idx + 12) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + handIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + if (compactString.Len() < idx + 18) return; + rootTranslation.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; } -void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const FString& compactString) { - if (compactString.Len() < 4) return; - pointHandLeft.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); +void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessLeftHand = handObj->GetNumberField("Open"); } -void FPoseAILiveValues::ProcessCompactVectorsHandRight(const FString& compactString) { - if (compactString.Len() < 4) return; - pointHandRight.Set(FixedB64pairToFloat(compactString[0], compactString[1]), - FixedB64pairToFloat(compactString[2], compactString[3])); +void FPoseAILiveValues::ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessRightHand = handObj->GetNumberField("Open"); } - void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { hasChanged = false; SetAndCheckForChange(scalars.VisTorso, isTorso, hasChanged); @@ -190,6 +331,7 @@ void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { const FString FPoseAILiveValues::fieldPointScreen = FString(TEXT("PointScreen")); +const FString FPoseAILiveValues::fieldThumbScreen = FString(TEXT("ThumbScreen")); void FPoseAIScalarStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody) { FJsonObjectConverter::JsonObjectToUStruct(scaBody.ToSharedRef(), this); @@ -206,7 +348,7 @@ void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ bodyHeight = verbose.Scalars.BodyHeight; - stableFeet = FMath::RoundToInt(verbose.Scalars.StableFoot); + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); stanceYaw = verbose.Scalars.StanceYaw * 180.0f; chestYaw = verbose.Scalars.ChestYaw * 180.0f; isCrouching = verbose.Scalars.IsCrouching > 0.5f; @@ -216,6 +358,11 @@ void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ upperBodyLean *= 180.0f; ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); + ProcessArrayAsVector3D(verbose.Vectors.Hip, rootTranslation); + ProcessArrayAsVector3D(verbose.Vectors.FootIkL, footIkL); + ProcessArrayAsVector3D(verbose.Vectors.FootIkR, footIkR); } @@ -224,11 +371,17 @@ void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonOb if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); } -void FPoseAILiveValues::ProcesssVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ +void FPoseAILiveValues::ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ if (vecHand==nullptr) return; ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); } + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp similarity index 85% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp index ace89a0..70554f9 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -1,24 +1,24 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "SPoseAILiveLinkWidget.h" #include "PoseAILiveLinkSourceFactory.h" -#include "PoseAILiveLinkSingleSource.h" +#include "PoseAILiveLinkNetworkSource.h" #define LOCTEXT_NAMESPACE "PoseAI" TWeakPtr SPoseAILiveLinkWidget::source = nullptr; +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; -static TArray PoseAI_Rigs = { "UE4", "MetaHuman", "Mixamo", "DazUE"}; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; -int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkSingleSource::portDefault; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; int32 SPoseAILiveLinkWidget::syncFPS = 60; int32 SPoseAILiveLinkWidget::cameraFPS = 60; int32 SPoseAILiveLinkWidget::modeIndex = 0; int32 SPoseAILiveLinkWidget::rigIndex = 0; bool SPoseAILiveLinkWidget::isMirrored = false; -bool SPoseAILiveLinkWidget::useRootMotion = true; bool SPoseAILiveLinkWidget::isIPv6 = false; BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION @@ -26,7 +26,6 @@ void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) { GConfig->GetBool(*section, TEXT("isIPv6"), isIPv6, GEditorIni); GConfig->GetBool(*section, TEXT("isMirrored"), isMirrored, GEditorIni); - GConfig->GetBool(*section, TEXT("useRootMotion"), useRootMotion, GEditorIni); GConfig->GetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); if (modeIndex < 0 || modeIndex >= PoseAI_Modes.Num()) @@ -122,20 +121,6 @@ void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) .IsChecked(isMirrored ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) ] ] - + SVerticalBox::Slot().AutoHeight() - [ - SNew(SHorizontalBox) - + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) - [ - SNew(STextBlock) - .Text(LOCTEXT("UseRootMotion", "Use root motion")) - ] - + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) - [ - SAssignNew(rootMotionCheckBox, SCheckBox) - .IsChecked(useRootMotion ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) - ] - ] + SVerticalBox::Slot().AutoHeight() [ SNew(SHorizontalBox) @@ -192,7 +177,7 @@ bool SPoseAILiveLinkWidget::IsPortValid() const return false; } - if (!PoseAILiveLinkSingleSource::IsValidPort(portNum)) { + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); return false; } @@ -234,22 +219,24 @@ void SPoseAILiveLinkWidget::disableExistingSource() TSharedPtr linkSource = source.Pin(); if (linkSource.IsValid()) { UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); - ((PoseAILiveLinkSingleSource*)linkSource.Get())->disable(); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); } } - - -TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() { FPoseAIHandshake handshake = FPoseAIHandshake(); handshake.isMirrored = isMirrored; - handshake.rig = PoseAI_Rigs[rigIndex]; - handshake.mode = PoseAI_Modes[modeIndex]; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); handshake.syncFPS = syncFPS; handshake.cameraFPS = cameraFPS; - handshake.useRootMotion = useRootMotion; UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); - return MakeShared(portNum, isIPv6, handshake); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); } FReply SPoseAILiveLinkWidget::OnToggleModeClicked() @@ -275,11 +262,9 @@ FReply SPoseAILiveLinkWidget::OnToggleRigClicked() FReply SPoseAILiveLinkWidget::OnOkClicked() { ReadCheckBox(mirroredCheckBox, isMirrored); - ReadCheckBox(rootMotionCheckBox, useRootMotion); ReadCheckBox(ipv6CheckBox, isIPv6); GConfig->SetBool(*section, TEXT("isMirror"), isMirrored, GEditorIni); - GConfig->SetBool(*section, TEXT("useRootMotion"), useRootMotion, GEditorIni); if (IsPortValid()) { GConfig->SetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); @@ -290,8 +275,7 @@ FReply SPoseAILiveLinkWidget::OnOkClicked() FLiveLinkLog::Info( TEXT("PoseAI: Setup source. Rig is in %s format, %s and %s."), *PoseAI_Rigs[rigIndex], - *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")), - *FString(useRootMotion ? TEXT("uses root motion"): TEXT("translate motion in the hip/pelvis bone")) + *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")) ); } return FReply::Handled(); @@ -302,4 +286,6 @@ void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& re TSharedPtr pin = checkBox.Pin(); if (pin) readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); -} \ No newline at end of file +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h similarity index 98% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h index df646d4..977dea1 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h similarity index 83% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h index bf095a7..3eecfa0 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -10,8 +10,6 @@ #include "PoseAIEventDispatcher.generated.h" -#define LOCTEXT_NAMESPACE "PoseAI" - DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); DECLARE_MULTICAST_DELEGATE_TwoParams(FPoseAIConfigUpdate, const FLiveLinkSubjectName&, FPoseAIModelConfig); @@ -33,7 +31,7 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAICrouchEvent, bool, isCrouchin DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIHandToZoneEvent, int32, zone); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmGestureEvent, int32, armGesture); DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIStationaryEvent); - +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIResetLivePositionEvent); UCLASS(ClassGroup = (PoseAI)) @@ -129,7 +127,7 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void ChangeModelConfig(FPoseAIModelConfig config); - /** sends disconnect message to app and clsoes source, freeing up Port*/ + /** sends disconnect message to app and closes source, freeing up Port*/ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void CloseSource(); @@ -141,6 +139,10 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") FLiveLinkSubjectName GetSubjectName() { return subjectName; } + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void RegisterAsFirstAvailable(); @@ -156,6 +158,24 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") void Deregister(); + /** Sets rig height, which scales root motion, and allows scaling per dimension (i.e. set Y=0 for no motion to/from camera) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ScaleMotion(float RigHeight=170.0f, FVector Scale=FVector(1.0f, 1.0f,1.0f)); + + /** compensates for an active source's camera rotation in the real world */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void SetLiveCameraRotation(float pitch, float yaw = 0.0f, float roll=0.0f); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseAsBaseTranslation(); + + /** Have player stand up straight and face screen as part of configuration */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseToOrientCamera(); + + /** Remove all live root motion (sets scalemotion to zero)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ZeroMotion(); UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") FDateTime lastFrameReceived; @@ -268,6 +288,8 @@ class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent private: FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + void InitializeObjects(); }; @@ -301,12 +323,34 @@ GENERATED_BODY() FPoseAIDisconnect disconnect; FPoseAIDisconnect closeSource; + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + /** Convenience event to tell animation blueprin*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + void BroadcastResetLivePosition(); + + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") FPoseAISubjectConnected subjectConnected; + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIResetLivePositionEvent resetLivePositionEvent; + // Connection driven events void BroadcastCloseSource(const FLiveLinkSubjectName& subjectName); void BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config); @@ -337,6 +381,7 @@ GENERATED_BODY() bool HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component); + void BroadcastResetZeroLivePosition(); private: static UPoseAIEventDispatcher* theInstance; diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h similarity index 80% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h index ef77995..75d849b 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021. All Rights Reserved. +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,6 +14,7 @@ class FPoseAILiveLinkModule : public IModuleInterface virtual void StartupModule() override; virtual void ShutdownModule() override; - - +private: + }; + diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..13d5dee --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,105 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..56ac13d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,72 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h similarity index 50% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index 77193e5..6fd81a0 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,9 +19,8 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" -#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkFaceSubSource.h" -#define LOCTEXT_NAMESPACE "PoseAI" struct POSEAILIVELINK_API PoseAIPortRecord { @@ -30,70 +29,116 @@ struct POSEAILIVELINK_API PoseAIPortRecord { FLiveLinkSubjectKey subjectKey; }; +class PoseAILiveLinkSingleSourceListener; + + /** * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to * trigger frame events and update the LiveLink pose source information. */ -class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource { public: - static const int32 portDefault = 8080; - PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake); + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); // standard Live Link source methods virtual bool CanBeDisplayedInUI() const { return true; } virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } - virtual FText GetSourceType() const { - return LOCTEXT("SourceType", "PoseAI mobile"); - } - virtual FText GetSourceMachineName() const { - return LOCTEXT("SourceMachineName", "Unreal Engine");; - } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; virtual FText GetSourceStatus() const { return status; } - virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) {} + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; virtual bool IsSourceStillValid() const override; virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} - virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid); + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; virtual bool RequestSourceShutdown(); - virtual void Update() override; - - - // method to add a source from code, instead of relying on presets. Exposed to blueprints via the PoseAI movement component. - static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + virtual void Update() override {} + // custom methods static bool GetPortGuid(int32 port, FGuid& fguid); static bool IsValidPort(int32 port); - - //Assigns a Livelink subject name from a port number. Returns FName as that class is used in LiveLinkSubjectKey consturctor - static FName SubjectNameFromPort(int32 port); - - //Looks up the "connection name" associated with port/subjetname which is the phone name @ IP Address. static FName GetConnectionName(int32 port); static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); - void SetConnectionName(FName name); - + static FName SubjectNameFromPort(int32 port); + void disable(); FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } - void UpdatePose(TSharedPtr rig, TSharedPtr jsonPose); + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - TSharedPtr udpServer; - - /* stores ports across different sources to avoid conflict from user input */ - static TMap usedPorts; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; - ILiveLinkClient* liveLinkClient = nullptr; - ILiveLinkClient* client = nullptr; - - UPoseAIEventDispatcher* dispatcher; + void AddSubject(); - mutable FText status; - - void AddSubject(TSharedPtr rig); +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + }; diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h similarity index 95% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h index 99a8847..1819982 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -1,4 +1,4 @@ -// Copyright PoseAI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once #include "CoreMinimal.h" diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h similarity index 72% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h index 6121ac4..c6d53c7 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -14,21 +14,22 @@ #include "Json.h" #include "PoseAIStructs.h" #include "PoseAIUdpSocketReceiver.h" -#include "PoseAIRig.h" #include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" -#define LOCTEXT_NAMESPACE "PoseAI" class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; class FPoseAISocketSender; -class PoseAILiveLinkSingleSource; +// The networking class needs to be rewritten class POSEAILIVELINK_API PoseAILiveLinkServer { - friend PoseAILiveLinkSingleSource; public: - PoseAILiveLinkServer(FPoseAIHandshake myHandshake, PoseAILiveLinkSingleSource* mySource, bool isIPv6 = false, int32 portNum = 8080); + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); ~PoseAILiveLinkServer() { CleanUp(); @@ -38,13 +39,14 @@ class POSEAILIVELINK_API PoseAILiveLinkServer static bool GetIP(FString& myIP); void CleanUp(); - void CloseTarget(const FLiveLinkSubjectName& target); void Disconnect(); - void DisconnectTarget(const FLiveLinkSubjectName& target); TSharedPtr GetSocket() const { return serverSocket; } - void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint); - void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config); + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; void SendHandshake() const; void SetHandshake(const FPoseAIHandshake& handshake); // receiver will be set on a runnable thread and set once started @@ -55,41 +57,38 @@ class POSEAILIVELINK_API PoseAILiveLinkServer const static FString fieldPrettyName; const static FString fieldVersion; const static FString fieldUUID; - static const FString fieldRigType; + const static FString fieldRigType; const static FString requiredMinVersion; - - PoseAILiveLinkSingleSource* source_ = nullptr; - + TSharedPtr listener; + TWeakPtr source_; FPoseAIHandshake handshake; FName protocolType; int32 port; + bool cleaningUp = false; // time of last connection. After timeout seconds a newer connection can takeover the port. FDateTime lastConnection; const double TIMEOUT_SECONDS = 10.0; - // dirty flag which source checks during update to see if rig static data needs updating - bool hasNewRig = false; - TSharedPtr serverSocket; - TSharedPtr rig; + //used to launch receiver without slowing main thread TSharedPtr poseAILiveLinkRunnable; + //Listens for packets TSharedPtr udpSocketReceiver; + //sends instructions to paired app TSharedPtr udpSocketSender; FPoseAIEndpoint endpoint; - - // disconnect message formatted for Pose AI mobile app FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + - bool SendString(FString& message) const; bool HasValidConnection() const; FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; @@ -101,30 +100,20 @@ class POSEAILIVELINK_API PoseAILiveLinkServer void CleanUpReceiver(); void CleanUpSender(); void CleanUpSocket(); - bool cleaningUp = false; + }; + + class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable { public: - PoseAILiveLinkReceiverRunnable(int32 port, PoseAILiveLinkServer* server) : - port(port), poseAILiveLinkServer(server) { + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); } - - virtual uint32 Run() override { - UE_LOG(LogTemp, Display, TEXT("PoseAI: Running server thread")); - FTimespan inWaitTime = FTimespan::FromMilliseconds(250); - FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); - udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, * receiverName); - udpSocketReceiver->OnDataReceived().BindRaw(poseAILiveLinkServer, &PoseAILiveLinkServer::ReceiveUDPDelegate); - udpSocketReceiver->Start(); - poseAILiveLinkServer->SetReceiver(udpSocketReceiver); - poseAILiveLinkServer = nullptr; - thread = nullptr; - return 0; - } + virtual uint32 Run() override; protected: FString myName; @@ -132,16 +121,19 @@ class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable FRunnableThread* thread = nullptr; private: PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; TSharedPtr udpSocketReceiver; }; + + // built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable { public: - FPoseAISocketSender(TSharedPtr socket, const TCHAR* threadDescription) : - socket(socket) { + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); } @@ -173,13 +165,14 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) { if (running) { + int32 sent = 0; - if (socket == nullptr) { + if (!Socket) { UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); return false; } - if (!socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); if (sent != Data->Num()) @@ -199,7 +192,7 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable protected: /** The network socket. */ - TSharedPtr socket; + TSharedPtr Socket; /** The thread object. */ FRunnableThread* thread = nullptr; @@ -208,3 +201,17 @@ class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable bool sleeping = false; }; + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h similarity index 58% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h index 61bbdc1..73f462b 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -7,8 +7,6 @@ #include "ILiveLinkSource.h" #include "PoseAILiveLinkSourceFactory.generated.h" -#define LOCTEXT_NAMESPACE "PoseAI" - /** * */ @@ -18,17 +16,15 @@ class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFa GENERATED_BODY() UPoseAILiveLinkSourceFactory() { - UE_LOG(LogTemp, Display, TEXT("instantiating PoseAILiveLinkSourceFactory")); } ~UPoseAILiveLinkSourceFactory () { - UE_LOG(LogTemp, Display, TEXT("destroying PoseAILiveLinkSourceFactory")); } public: virtual TSharedPtr< SWidget > BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const override; virtual TSharedPtr< ILiveLinkSource > CreateSource(const FString& ConnectionString) const override; - virtual FText GetSourceDisplayName() const override { return LOCTEXT("Pose AI App", "Pose AI App"); } - virtual FText GetSourceTooltip() const override { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + virtual FText GetSourceDisplayName() const override; + virtual FText GetSourceTooltip() const override; virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } }; diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h similarity index 85% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h index 6d3f035..1e8e6ac 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -10,8 +10,6 @@ #include "Json.h" #include "PoseAIStructs.h" -#define LOCTEXT_NAMESPACE "PoseAI" - struct POSEAILIVELINK_API Remapping { FName TargetJointName; @@ -27,19 +25,26 @@ struct POSEAILIVELINK_API Remapping class POSEAILIVELINK_API PoseAIRig { public: - FLiveLinkStaticDataStruct rig; FLiveLinkStaticDataStruct MakeStaticData(); bool ProcessFrame(const TSharedPtr, FLiveLinkAnimationFrameData& data); static bool IsFrameData(const TSharedPtr jsonObject); static TSharedPtr PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake); + static TWeakPtr GetRigFromSubjectName(const FLiveLinkSubjectName& name); FName RigType() { return rigType; } - + FPoseAIVisibilityFlags visibilityFlags; FPoseAILiveValues liveValues; FPoseAIScalarStruct scalars; FPoseAIEventStruct events; + bool useNextRootAsOffset = false; + //ankle to head top height for scaling PoseAI root motion. + float rigHeight = 170.0f; + + float CameraTilt = 0.0f; + protected: + FLiveLinkStaticDataStruct rig; FPoseAIVerbose verbose; static const FString fieldBody; static const FString fieldRigType; @@ -61,21 +66,24 @@ class POSEAILIVELINK_API PoseAIRig FLiveLinkSubjectName name; FName rigType; - bool useRootMotion; + bool includeHands; bool isMirrored; + bool isLowerBodyRotated; bool isDesktop; int32 numBodyJoints = 21; int32 numHandJoints = 17; // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) int32 lowerBodyNumOfJoints = 8; + int32 rShinJoint = 3; + int32 lShinJoint = 7; bool isCrouching = false; int32 handZoneL = 5; int32 handZoneR = 5; int32 stableFeet = 0; - + FVector prevRootTranslation = FVector::ZeroVector; // store translations of deployed rig TMap boneVectors; TArray jointNames; @@ -84,24 +92,26 @@ class POSEAILIVELINK_API PoseAIRig //temporary variable used for convenience in rig construction FName lastBoneAdded; - - //ankle to head top height for scaling PoseAI root motion. - float rigHeight = 170.0f; //extra offset for hip bone to accomodate mesh thickness from bone sockets. - float rootHipOffsetZ = 1.0f; + float rootHipOffsetZ = 2.0f; void AddBone(FName boneName, FName parentName, FVector translation); void AddBoneToLast(FName boneName, FVector translation); void CachePose(const TArray& transforms); void AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data); void AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); bool ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); bool ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); void TriggerEvents(); - void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + void RotateLowerBody180(TArray& quatArray); + + +private: + static TMap> RigMap; }; class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { @@ -118,6 +128,14 @@ class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { void Configure(); }; +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { public: PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; @@ -130,4 +148,4 @@ class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; protected: void Configure(); -}; \ No newline at end of file +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h similarity index 67% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h index 78a665f..32ae5d2 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -8,9 +8,6 @@ #include "PoseAIStructs.generated.h" -#define LOCTEXT_NAMESPACE "PoseAI" - - /* decoding utilities for compact representation */ float UintB64ToUint(char a, char b); @@ -19,64 +16,122 @@ float FixedB64pairToFloat(char a, char b); void FStringFixed12ToFloat(const FString& data, TArray& flatArray); void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + /* the handshake configures the main parameters of pose camera*/ USTRUCT(BlueprintType) -struct POSEAILIVELINK_API FPoseAIHandshake +struct POSEAILIVELINK_API FPoseAIHandshake { GENERATED_BODY() - - /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString mode = "Room"; + EPoseAiAppModes mode = EPoseAiAppModes::Room; /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString rig = "UE4"; + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; - /* the model context. Will enable new AI models as they are deployed*/ + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString context = "Default"; + bool isFaceAnimating = true; - /* the target frame rate, where phone does interpolation and smoothing for animations. Events are raw.*/ + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 syncFPS = 60; + bool isMirrored = true; - /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 cameraFPS = 60; + bool isLowerBodyRotated = false; - /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool isMirrored = false; - - /* controls compactness of packet. 0 is verbose JSON (mainly use for debugging), 1 is fairly compact JSON (preferred as of this plugin release). We may add even more condensed formats in the future.*/ + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - int32 packetFormat = 1; + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; - /* whether to include motion within camera frame in hips or in root*/ + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - bool useRootMotion = false; + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; - /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString whoami = ""; + FString whoami = ""; - /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") - FString signature = ""; + FString signature = ""; + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; - FString ToString() const; + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + bool operator==(const FPoseAIHandshake& Other) const; - bool operator!=(const FPoseAIHandshake& Other) const {return !operator==(Other);} + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; FString YesNoString(bool val) const { return val ? FString("YES") : FString("NO"); } }; + /*adjusts the sensitivity of PoseAI events*/ USTRUCT(BlueprintType) struct POSEAILIVELINK_API FPoseAIModelConfig @@ -100,7 +155,7 @@ struct POSEAILIVELINK_API FPoseAIModelConfig float jumpSensitivity = 0.5f; UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") - bool isMirrored; + bool isMirrored = true; FString ToString() const; FString YesNoString(bool val) const { @@ -232,6 +287,9 @@ struct POSEAILIVELINK_API FPoseAIScalarStruct UPROPERTY(BlueprintReadOnly, Category = "Flags") float VisLegR = 0.0f; + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisFace = 0.0f; + /** location of left hand relative to body in broad zones */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") int32 HandZoneL = 5; @@ -277,6 +335,16 @@ struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors TArray HipScreen; UPROPERTY() TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; + UPROPERTY() + TArray Hip; + UPROPERTY() + TArray FootIkL; + UPROPERTY() + TArray FootIkR; }; @@ -318,6 +386,8 @@ struct POSEAILIVELINK_API FPoseAIVisibilityFlags UPROPERTY(BlueprintReadOnly, Category = "Flags") bool isRightLeg = false; + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isFace = false; bool HasChanged() { return hasChanged; } void ProcessVerbose(FPoseAIScalarStruct& scalars); @@ -335,6 +405,7 @@ struct POSEAILIVELINK_API FPoseAILiveValues { GENERATED_BODY() public: + /** How much subject is leaning in radians, head-to-hips; x to the side, y forward */ UPROPERTY(BlueprintReadOnly, Category="PoseAI") FVector2D upperBodyLean = FVector2D(0.0f, 0.0f); @@ -351,9 +422,9 @@ GENERATED_BODY() UPROPERTY(BlueprintReadOnly, Category = "PoseAI") int32 handZoneRight = 5; - /** offset location of subject in camera frame. Good for lateral movement */ + /** location of subject in camera frame, scaled in body units (i.e. multiply by rig height to translate to game world). Pos Y moves toward camera */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") - FVector rootTranslation = FVector::ZeroVector; + FVector rootTranslation = FVector(0.0f, -2.0f, 0.0f); /** Heading in radians of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") @@ -361,22 +432,21 @@ GENERATED_BODY() /** Heading in radians of torso. 0 is heading to camera */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") - float chestYaw = 0.0f; - + float chestYaw = 0.0f; /** Heading of flattened left foot to right foot vector relative to camera */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") - int32 modelLatency = 0.0f; + int32 modelLatency = 0.0f; /** timestamp according to pose camera device (CMTime), in seconds */ double timestamp = 0.0; - /** current height of jump in body units */ - UPROPERTY(BlueprintReadOnly, Category = "PoseAI") - float jumpHeight = 0.0f; + /** DEPR current height of jump in body units */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI_DEPR") + float jumpHeight = 0.0f; - /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ - UPROPERTY(BlueprintReadOnly, Category="PoseAI") + /** DEPR estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI_DEPR") float bodyHeight = 0.0f; /** location of hips in camera frame (clip coordinates -1 to 1, 0 is center ) */ @@ -395,24 +465,65 @@ GENERATED_BODY() UPROPERTY(BlueprintReadOnly, Category="PoseAI") FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + /** location of left thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbLeft = FVector2D(0.0f, 0.0f); + + /** location of right thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbRight = FVector2D(0.0f, 0.0f); + + /** how open the left hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessLeftHand = 0.5f; + + /** how open the right hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessRightHand = 0.5f; + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); /** if at least one foot has been stationary for a few frames */ UPROPERTY(BlueprintReadOnly, Category = "PoseAI") int32 stableFeet = 0; - + + FVector rootOffset = FVector(0.0f, 0.0f, 0.0f); + FRotator cameraRotation = FRotator(0.0f, 0.0f, 0.0f); + FVector scaleMotion = FVector(1.0f, 0.0f, 1.0f); void ProcessVerboseBody(const FPoseAIVerbose& scalars); void ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand); - void ProcesssVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); + void ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); void ProcessCompactScalarsBody(const FString& compactString); void ProcessCompactVectorsBody(const FString& compactString); - void ProcessCompactVectorsHandLeft(const FString& compactString); - void ProcessCompactVectorsHandRight(const FString& compactString); - + void ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject >); + void ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject >); private: static const FString fieldPointScreen; -}; - + static const FString fieldThumbScreen; +}; diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h similarity index 86% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h index df42fe5..9434a67 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -1,4 +1,4 @@ -// Pose AI Ltd Copyright 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. // This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) #pragma once @@ -58,7 +58,7 @@ class FPoseAIUdpSocketReceiver { check(Socket != nullptr); check(Socket->GetSocketType() == SOCKTYPE_Datagram); - + Reader->SetNumUninitialized(MaxReadBufferSize); SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); } @@ -138,6 +138,7 @@ class FPoseAIUdpSocketReceiver /** Update this socket receiver. */ void Update(const FTimespan& SocketWaitTime) { + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) { return; @@ -151,26 +152,30 @@ class FPoseAIUdpSocketReceiver TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); uint32 Size; - - while (Socket!=nullptr && Socket.IsValid() && Socket->HasPendingData(Size)) + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) { - Reader->SetNumUninitialized(FMath::Min(Size, MaxReadBufferSize)); - - int32 Read = 0; - if (Socket->RecvFrom(Reader->GetData(), Reader->Num(), Read, *Sender)) + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) { - ensure((uint32)Read < MaxReadBufferSize); - Reader->RemoveAt(Read, Reader->Num() - Read, false); - // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs - FString recvMessage; - char* bytedata = (char*)Reader->GetData(); - bytedata[Reader->Num()] = '\0'; - recvMessage = FString(UTF8_TO_TCHAR(bytedata)); + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); } + } - + } protected: diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h similarity index 96% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h rename to UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h index d74a39b..dfa1571 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -1,4 +1,4 @@ -// Copyright Pose AI 2021. All rights reserved +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -52,10 +52,9 @@ class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public static int32 modeIndex; static int32 rigIndex; static bool isMirrored; - static bool useRootMotion; static bool isIPv6; - + static FPoseAIHandshake GetHandshake(); void UpdatePort(const FText& InText, ETextCommit::Type type); void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs new file mode 100644 index 0000000..e39bf7b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -0,0 +1,60 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLinkEd : ModuleRules +{ + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.0/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/PoseAILiveLink.uplugin new file mode 100644 index 0000000..680790f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/PoseAILiveLink.uplugin @@ -0,0 +1,39 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "3.0.2", + "FriendlyName": "PoseAI LiveLink", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", + "Category": "Animation", + "CreatedBy": "Pose AI Ltd", + "CreatedByURL": "www.pose-ai.com", + "DocsURL": "www.pose-ai.com/unreal-engine-livelink", + "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/3db8f23ad003492eb85a50de31e5bc57", + "SupportURL": "support@pose-ai.com", + "EngineVersion": "5.1.0", + "CanContainContent": false, + "IsBetaVersion": false, + "Installed": true, + "Modules": [ + { + "Name": "PoseAILiveLink", + "Type": "Runtime", + "LoadingPhase": "Default", + "WhitelistPlatforms": [ + "Win64", + "Mac" + ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "LiveLink", + "Enabled": true + } + ] +} \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Resources/Icon128.png new file mode 100644 index 0000000..a5f1494 Binary files /dev/null and b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Resources/Icon128.png differ diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..6042e8f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,107 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp new file mode 100644 index 0000000..6a2d82f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -0,0 +1,43 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEndpoint.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port) { + + FName socketType = NAME_DGram; + FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + + socket->SetReuseAddr(); + int actualSize; + socket->SetReceiveBufferSize(64 * 1024, actualSize); + socket->SetSendBufferSize(64 * 1024, actualSize); + + TSharedRef sender = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr(protocolType); + sender->SetIp(0); + sender->SetPort(port); + socket->Bind(*sender); + return MakeShareable(socket); +} + + +FString FPoseAIEndpoint::ToString() const +{ + return Address->ToString(true); +} + + +void FPoseAIEndpoint::Initialize() +{ + CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp new file mode 100644 index 0000000..440a2df --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -0,0 +1,468 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +void UStepCounter::Halt(bool fade) { + num_ = 0; + tail_ = -1; + if (fade) + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout))); + else { + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout + fadeDurationOnTimeout))); + steps_per_second_ = 0.0f; + distance_per_second_ = 0.0f; + } +} + +float UStepCounter::CheckIfActiveAndFade() { + float time_since_last_step = TimeSinceLastStep(); + if (time_since_last_step > timeout) { + num_ = 0; + tail_ = -1; + return (fadeDurationOnTimeout > 0.0f) ? + FMath::Max(0.0f, 1.0f - (time_since_last_step - timeout) / fadeDurationOnTimeout) + : 0.0f; + } + else + return 1.0f; +} + +float UStepCounter::StepsPerSecond() { + return steps_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::DistancePerSecond() { + return distance_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::LastDistance() { + return (num_ > 0) ? heights_[tail_] : 0.0f; +} + +float UStepCounter::TimeSinceLastStep() { + return (FDateTime::Now() - last_time_).GetTotalSeconds();; +} + +void UStepCounter::RegisterStep(float stepDistance) { + totalSteps++; + totalDistance += stepDistance; + tail_ = (tail_ + 1) % num_to_track; + last_time_ = FDateTime::Now(); + times_[tail_] = last_time_; + heights_[tail_] = stepDistance; + num_ = FGenericPlatformMath::Min(num_ + 1, num_to_track); + + float elapsed_time; + if (num_ < 2) + elapsed_time = 1.0f; + else { + int head_ = (tail_ + 1) % num_; + elapsed_time = (times_[tail_] - times_[head_]).GetTotalSeconds(); + if (elapsed_time <= 0.0) elapsed_time = 1.0f; + } + steps_per_second_ = static_cast(times_.Num()) / elapsed_time; + distance_per_second_ = 0.0f; + for (int h = 0; h < num_; ++h) + distance_per_second_ += heights_[h]; + distance_per_second_ /= elapsed_time; + +} +UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { + timeout = timeoutIn; + fadeDurationOnTimeout = fadeDuration; + return this; +} + + +bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, myIP, portNum, isIPv6); +} + +bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { + InitializeObjects(); + FLiveLinkSubjectName addedSubjectName; + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); +} + +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} + +void UPoseAIMovementComponent::InitializeObjects() { + footsteps = NewObject(); + leftsteps = NewObject()->SetProperties(0.3f, 0.1f); + rightsteps = NewObject()->SetProperties(0.3f, 0.1f); + feetsplits = NewObject(); + armpumps = NewObject(); + armflexes = NewObject(); + armjacks = NewObject()->SetProperties(1.0f, 0.5f); + armflapL = NewObject()->SetProperties(0.5f, 1.5f); + armflapR = NewObject()->SetProperties(0.5f, 1.5f); + jumps = NewObject(); +} + +bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken) { + InitializeObjects(); + bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); + if (success) + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); + return success; +} + +void UPoseAIMovementComponent::RegisterAsFirstAvailable() { + InitializeObjects(); + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentForFirstAvailableSubject(this); +} + +void UPoseAIMovementComponent::CloseSource() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCloseSource(subjectName); +} + +void UPoseAIMovementComponent::Disconnect() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastDisconnect(subjectName); +} + +void UPoseAIMovementComponent::Deregister() { + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, FLiveLinkSubjectName(NAME_None) ,true); + subjectName = FLiveLinkSubjectName(NAME_None); +} + +void UPoseAIMovementComponent::ChangeModelConfig(FPoseAIModelConfig config) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastConfigUpdate(subjectName, config); +} + +void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { + UPoseAIEventDispatcher::GetDispatcher()->SetHandshake(handshake); +} + +void UPoseAIMovementComponent::ScaleMotion(float RigHeight, FVector Scale) { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->rigHeight = RigHeight; + lockedRig->liveValues.scaleMotion = Scale; + } +} + +void UPoseAIMovementComponent::SetLiveCameraRotation(float pitch, float yaw, float roll){ + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + // order changed due to rotated root bone in UE + lockedRig->liveValues.cameraRotation = FRotator(roll, yaw, pitch); + } +} + + +void UPoseAIMovementComponent::UseCurrentPoseToOrientCamera() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig==nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + float pitch = -lockedRig->liveValues.upperBodyLean.Y; + float yaw = -lockedRig->liveValues.chestYaw; + lockedRig->liveValues.cameraRotation = FRotator(0.0f, yaw, pitch); + } +} +void UPoseAIMovementComponent::UseCurrentPoseAsBaseTranslation() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.rootOffset = lockedRig->liveValues.rootTranslation; + } +} + +void UPoseAIMovementComponent::ZeroMotion() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.scaleMotion = FVector3d::Zero(); + } +} + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + +bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); + LastMovementComponent = component; + UPoseAIMovementComponent* existing_component; + if (HasComponent(name, existing_component)) { + if (!siezeIfTaken) return false; + existing_component->Deregister(); + } + + FName prev = component->GetSubjectName(); + componentsByName.Remove(prev); + if (name.Name != NAME_None) + componentsByName.Emplace(name, component); + component->lastFrameReceived = FDateTime::Now(); + component->subjectName = name; + return true; +} + + +UPoseAIEventDispatcher* UPoseAIEventDispatcher::theInstance = nullptr; + + +void UPoseAIEventDispatcher::RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component) { + FLiveLinkSubjectName firstUnbound = GetFirstUnboundSubject(); + if (firstUnbound.Name != NAME_None) + component->RegisterAs(firstUnbound, true); + else + componentQueue.Enqueue(component); +} + +FLiveLinkSubjectName UPoseAIEventDispatcher::GetFirstUnboundSubject(bool excludeIdleSubjects) { + for (auto& elem : knownConnectionsWithTime) { + if (excludeIdleSubjects && (FDateTime::Now() - elem.Value).GetTotalSeconds() > timeoutInSeconds) continue; + UPoseAIMovementComponent* component; + if (HasComponent(elem.Key, component) && component->GetSubjectName() == elem.Key) continue; + return elem.Key; + } + return NAME_None; +} + + +void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName) { + knownConnectionsWithTime.Emplace(subjectName, FDateTime::Now()); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* existing_component; + bool isReconnection = HasComponent(subjectName, existing_component); + if (isReconnection) { + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); + } else if (!componentQueue.IsEmpty()) { + UPoseAIMovementComponent* component; + componentQueue.Dequeue(component); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } + } + subjectConnected.Broadcast(subjectName, isReconnection); + }); +} + +void UPoseAIEventDispatcher::BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config) { + modelConfigUpdate.Broadcast(subjectName, config); +} + +void UPoseAIEventDispatcher::BroadcastDisconnect(const FLiveLinkSubjectName& subjectName) { + disconnect.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastCloseSource(const FLiveLinkSubjectName& subjectName) { + closeSource.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastResetLivePosition() { + resetLivePositionEvent.Broadcast(); +} +void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { + handshakeUpdate.Broadcast(handshake); +} + +void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); + }); +} + +void UPoseAIEventDispatcher::BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, visibilityFlags]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onVisibilityChange.Broadcast(visibilityFlags); + component->visibilityFlags = visibilityFlags; + } + }); +} + +void UPoseAIEventDispatcher::BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, values]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onLiveValues.Broadcast(values); + component->SetLiveValues(values); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->footsteps->RegisterStep(stepHeight); + component->onFootstep.Broadcast(stepHeight, isLeftStep); + } + }); +} +void UPoseAIEventDispatcher::BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->feetsplits->RegisterStep(width); + component->onFeetsplit.Broadcast(width, isExpanding); + } + }); +} +void UPoseAIEventDispatcher::BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armpumps->RegisterStep(stepHeight); + component->onArmpump.Broadcast(stepHeight); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armflexes->RegisterStep(width); + component->onArmflex.Broadcast(width, isExpanding); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isRising]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armjacks->RegisterStep(0.5f); + component->onArmjack.Broadcast(isRising); + } + }); +} + + +void UPoseAIEventDispatcher::BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepLeftFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepRightFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastJumps(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onJump.Broadcast(); + component->jumps->RegisterStep(1.0f); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isCrouching]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onCrouch.Broadcast(isCrouching); + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureLeft.Broadcast(gesture); + if (gesture == 10) { + component->armflapL->RegisterStep(1.0f); + component->onArmflapL.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureRight.Broadcast(gesture); + if (gesture == 10) { + component->armflapR->RegisterStep(1.0f); + component->onArmflapR.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneL.Broadcast(zone); + }); +} + + +void UPoseAIEventDispatcher::BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneR.Broadcast(zone); + }); +} + +void UPoseAIEventDispatcher::BroadcastStationary(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onStationary.Broadcast(); + }); +} + + +bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component) { + if (!componentsByName.Contains(name)) + return false; + component = componentsByName[name]; + return component != nullptr && IsValid(component); +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..fbc882c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..004c575 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + if (jsonPose != nullptr && jsonPose->HasField("Face")) { + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..31b50b0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,169 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..f3dd836 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp new file mode 100644 index 0000000..7c6b37e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -0,0 +1,82 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkRetargetRotations.h" + +#include "BonePose.h" +#include "BoneContainer.h" +#include "Engine/Blueprint.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "LiveLinkTypes.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "Roles/LiveLinkAnimationTypes.h" + + +UPoseAILiveLinkRetargetRotations::UPoseAILiveLinkRetargetRotations(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +#if WITH_EDITOR + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + if (Blueprint) + { + OnBlueprintCompiledDelegate = Blueprint->OnCompiled().AddUObject(this, &UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled); + } +#endif +} + +void UPoseAILiveLinkRetargetRotations::BeginDestroy() +{ +#if WITH_EDITOR + if (OnBlueprintCompiledDelegate.IsValid()) + { + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + check(Blueprint); + Blueprint->OnCompiled().Remove(OnBlueprintCompiledDelegate); + OnBlueprintCompiledDelegate.Reset(); + } +#endif + + Super::BeginDestroy(); +} + +void UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled(UBlueprint* TargetBlueprint) +{ + +} + + +void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) +{ + auto rootTransform = InFrameData->Transforms[0]; + auto rootTranslation = rootTransform.GetTranslation() * scaleTranslation; + auto rootZ = FVector3d(0.0f, 0.0f, rootTranslation.Z); + auto rootXY = FVector3d(rootTranslation.X, rootTranslation.Y, 0.0f); + for (int32 i = 0; i < InSkeletonData->BoneNames.Num(); i++) + { + FName boneName = InSkeletonData->BoneNames[i]; + auto jointTransform = InFrameData->Transforms[i]; + const int32 meshIndex = OutPose.GetBoneContainer().GetPoseBoneIndexForBoneName(boneName); + if (meshIndex != INDEX_NONE) + { + FCompactPoseBoneIndex boneIndex = OutPose.GetBoneContainer().MakeCompactPoseIndex(FMeshPoseBoneIndex(meshIndex)); + if (i == 0) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootXY); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + + else if (i == 1) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootZ); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + else if (boneIndex != INDEX_NONE) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + } + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp new file mode 100644 index 0000000..ff8da5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -0,0 +1,271 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkServer.h" +#include "Async/Async.h" +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); +const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); +const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); +const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); +const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); + + +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; + + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); + + FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); + FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); + serverSocket = BuildUdpSocket(serverName, protocolType, port); + poseAILiveLinkRunnable = MakeShared(port, listener, this); + udpSocketSender = MakeShared(serverSocket, *senderName); + + FString myIP; + if (protocolType == FNetworkProtocolTypes::IPv6) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on IPv6 link-Local address (begins with fe80:) and Port:%d"), port); + } else if (GetIP(myIP)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on %s Port:%d"), *myIP, port); + } else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server but can't determine your IP address. You may not have a valid network adapter.")); + } +} + +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + + +bool PoseAILiveLinkServer::GetIP(FString& myIP) { + bool canBind = false; + TSharedRef localIp = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->GetLocalHostAddr(*GLog, canBind); + + if (localIp->IsValid()) { + myIP = (localIp->ToString(false)); + return true; + } else { + myIP = LOCTEXT("undeterminedIP", "Can't determine your local host IP address").ToString(); + return false; + } +} + +void PoseAILiveLinkServer::CleanUp() { + if (!cleaningUp) { + cleaningUp = true; + CleanUpReceiver(); + CleanUpSender(); + TSharedPtr socketForClosure = serverSocket; + uint32 portnum = port; + // using delayed closure to try to resolve issue where Epic built plugin creates crashes while project plugin doesn't. + Async(EAsyncExecution::Thread, [socketForClosure, portnum]() { + FPlatformProcess::Sleep(2.0); + socketForClosure->Close(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Closed socket on Port:%d"), portnum); + }); + } +} + +void PoseAILiveLinkServer::CleanUpReceiver() { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); + udpSocketReceiver->Stop(); + } + + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); + poseAILiveLinkRunnable.Reset(); + } +} +void PoseAILiveLinkServer::CleanUpSender() { + if (udpSocketSender && udpSocketSender.IsValid()) { + Disconnect(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); + udpSocketSender->Stop(); + udpSocketSender->Exit(); + } +} + + +bool PoseAILiveLinkServer::HasValidConnection() const { + return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; +} + +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + if (cleaningUp) return; + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); + return; + } + + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); + if (HasValidConnection() && !sameAsCurrent) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { + endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint + SendHandshake(); + } + else { //reject + UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); + //consider sending rejected connection a warning message + } + } + else if (!HasValidConnection() && !sameAsCurrent) { // new connection + InitiateConnection(jsonObject, endpointRecv); + + } + else { + if (PoseAIRig::IsFrameData(jsonObject)) { + lastConnection = FDateTime::Now(); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } + } + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message + SendHandshake(); + } + } +} + +void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + FString version; + if (!(jsonObject->TryGetStringField(fieldVersion, version))) { + static const FName NAME_NoAppVersion = "PoseAILiveLink_NoAppVersion"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_NoAppVersion, failKey, TEXT("PoseAI: Incoming connection does not have a hello handshake")); + return; + } + else if (!CheckAppVersion(version)) { + static const FName NAME_AppVersionFail = "PoseAILiveLink_AppVersionFail"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_AppVersionFail, failKey, TEXT("PoseAI: Please update the mobile app to at least version %s"), *requiredMinVersion); + UE_LOG(LogTemp, Error, TEXT("PoseAILiveLink: Unknown app version. Can not safely connect."), *version); + return; + } + FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); + UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } +} + +bool PoseAILiveLinkServer::SendString(FString& message) const { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { + return false; + } +} + + +void PoseAILiveLinkServer::SendHandshake() const { + FString message_string = handshake.ToString(); + if (SendString(message_string)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); + } else { //unsuccessful + static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; + static const FGuid GUID_HandshakeFail = FGuid(); + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); + FLiveLinkLog::WarningOnce(NAME_HandshakeFail, failKey, TEXT("PoseAI: Unable to send the handshake to %s"), *(endpoint.ToString())); + } +} + +void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { + handshake = newHandshake; + if (endpoint.IsValid()) + SendHandshake(); +} + + + +void PoseAILiveLinkServer::Disconnect() { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*disconnect); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + if (udpSocketSender->Send(bytedata, endpoint)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent disconnect request to %s"), *(endpoint.ToString())); + } + else { //unsuccesful + UE_LOG(LogTemp, Display, TEXT("PoseAI: Unable to disconnect from %s"), *(endpoint.ToString())); + } + } +} + + +FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const +{ + FString prettyString; + if (!(jsonObject->TryGetStringField(fieldPrettyName, prettyString))) { + prettyString = "Unknown"; + } + return FName(*(prettyString.Append("@").Append(endpointRecv.Address->ToString(false)))); +} + +bool PoseAILiveLinkServer::CheckAppVersion(FString version) const +{ + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: App version %s vs required version %s."), *version, *requiredMinVersion); + TArray appArray; + version.ParseIntoArray(appArray, TEXT("."), false); + if (appArray.Num() < 3) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: App version %s format error"), *version); + } + + TArray requiredArray; + requiredMinVersion.ParseIntoArray(requiredArray, TEXT("."), false); + + for (int32 i = 0; i < 3; i++) { + int32 app = FCString::Atoi(*appArray[i]); + int32 req = FCString::Atoi(*requiredArray[i]); + if (app < req) + return false; + else if (app > req) + return true; + } + return true; +} + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp new file mode 100644 index 0000000..3eddf9f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -0,0 +1,25 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkSourceFactory.h" +#include "SPoseAILiveLinkWidget.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +TSharedPtr UPoseAILiveLinkSourceFactory::BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const +{ + auto rawWidget = SNew(SPoseAILiveLinkWidget); + rawWidget->setCallback(OnLiveLinkSourceCreated); + TSharedPtr myWidget = rawWidget; + return myWidget; +} + +TSharedPtr< ILiveLinkSource > UPoseAILiveLinkSourceFactory::CreateSource(const FString & ConnectionString) const +{ + return SPoseAILiveLinkWidget::CreateSource(ConnectionString); +} + +FText UPoseAILiveLinkSourceFactory::GetSourceDisplayName() const { return LOCTEXT("Pose AI App", "Pose AI App"); } +FText UPoseAILiveLinkSourceFactory::GetSourceTooltip() const { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp similarity index 84% rename from UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp rename to UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp index 6e85cb3..ef6cf16 100644 --- a/UnrealEngineAPI/PluginV1.3/4.27/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -1,8 +1,10 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #include "PoseAIRig.h" #include "PoseAIEventDispatcher.h" +#define LOCTEXT_NAMESPACE "PoseAI" + const FString PoseAIRig::fieldBody = FString(TEXT("Body")); const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); @@ -12,7 +14,7 @@ const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); - +TMap> PoseAIRig::RigMap = {}; bool isDifferentAndSet(int32 newValue, int32& storedValue) { bool isDifferent = newValue != storedValue; @@ -23,34 +25,45 @@ bool isDifferentAndSet(int32 newValue, int32& storedValue) { PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : name(name), - rigType(FName(handshake.rig)), - useRootMotion(handshake.useRootMotion), - includeHands(!handshake.mode.Contains(TEXT("BodyOnly"))), + rigType(FName(handshake.GetRigString())), + includeHands(handshake.IncludesHands()), isMirrored(handshake.isMirrored), - isDesktop(handshake.mode.Contains(TEXT("Desktop"))) { + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { Configure(); } - TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { TSharedPtr rigPtr; - FName rigType = FName(handshake.rig); - if (rigType == FName("Mixamo")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("MetaHuman")) { - rigPtr = MakeShared(name, handshake); - } - else if (rigType == FName("DazUE")) { - rigPtr = MakeShared(name, handshake); - } - else { - rigPtr = MakeShared(name, handshake);; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; } + rigPtr->Configure(); + RigMap.Add(name, rigPtr); return rigPtr; } +TWeakPtr PoseAIRig::GetRigFromSubjectName(const FLiveLinkSubjectName& name) { + return RigMap.Contains(name)? RigMap[name] : nullptr; +} + + FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ FLiveLinkStaticDataStruct staticData; staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); @@ -83,6 +96,7 @@ bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) { + double timestamp; jsonObject->TryGetNumberField("Timestamp", timestamp); // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. @@ -93,7 +107,11 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink FString rigStringOut; if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { - UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } return false; } @@ -105,13 +123,6 @@ bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLink else ProcessVerboseSupplementaryData(jsonObject, data); - if (visibilityFlags.isTorso && liveValues.bodyHeight > 0.0f) - liveValues.rootTranslation = FVector( - -liveValues.hipScreen[0] * rigHeight / liveValues.bodyHeight, //x is left in Unreal so flip - 0.0f, //currently no body distance estimate from pose camera - 0.0f - ); - TriggerEvents(); data.WorldTime = FPlatformTime::Seconds(); @@ -203,45 +214,19 @@ void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr js } objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; if (objHandLeft != nullptr && objHandLeft.IsValid()) { - FString VecA = (objHandLeft->HasTypedField("VecA")) ? objHandLeft->GetStringField("VecA") : ""; - liveValues.ProcessCompactVectorsHandLeft(VecA); + liveValues.ProcessCompactVectorsHandLeft(objHandLeft); } objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; if (objHandRight != nullptr && objHandRight.IsValid()) { - FString VecA = (objHandRight->HasTypedField("VecA")) ? objHandRight->GetStringField("VecA") : ""; - liveValues.ProcessCompactVectorsHandRight(VecA); + liveValues.ProcessCompactVectorsHandRight(objHandRight); } } void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { - - // to ensure grounding in the capsule, calculates lowest Z in component space. doesn't check fingers to save calculations on fingers: if this is important consider using parents.Num() - TArray componentTransform; - componentTransform.Emplace(data.Transforms[0]); - componentTransform.Emplace(data.Transforms[1]); - FVector baseTranslation; - float minZ = 0.0f; - if (isDesktop) - baseTranslation = FVector(0.0f, 0.0f, rigHeight * 0.5f); - else { - baseTranslation = liveValues.rootTranslation; //careful as this assumes liveValues has been updated already this frame - for (int32 j = 2; j < numBodyJoints; j++) { - componentTransform.Emplace(data.Transforms[j] * componentTransform[parentIndices[j]]); - minZ = FGenericPlatformMath::Min(minZ, componentTransform[j].GetTranslation().Z); - } - } - minZ -= rootHipOffsetZ; - - // assigns motion either to root or to hips - if (useRootMotion) { - //hip to low point Z distance assigned to hips, rest of movement assigned to root - data.Transforms[1].SetTranslation(FVector(0.0f, 0.0f, -minZ)); - data.Transforms[0].SetTranslation(baseTranslation); - } - else { - baseTranslation.Z -= minZ; - data.Transforms[1].SetTranslation(baseTranslation); + if (!isDesktop) { + FVector playerMotion = liveValues.cameraRotation.RotateVector(liveValues.rootTranslation - liveValues.rootOffset) * rigHeight * liveValues.scaleMotion; + data.Transforms[0].SetTranslation(playerMotion); } } @@ -285,11 +270,15 @@ bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject TArray quatArray; FStringFixed12ToFloat(rotaBody, flatArray); FlatArrayToQuats(flatArray, quatArray); - AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as psoe camera does not include the root joint + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint } else AppendCachedRotations(1, numBodyJoints, componentRotations, data); + if (includeHands) { if (rotaHandLeft.Len() > 7) { TArray flatArray; @@ -376,7 +365,6 @@ bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations data.Transforms.Add(transform); } - AssignCharacterMotion(data); CachePose(data.Transforms); @@ -407,21 +395,34 @@ void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr js //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; } + liveValues.ProcessVerboseBody(verbose); + if (objHandLeft != nullptr && objHandLeft.IsValid()) { vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; - + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + if (objHandLeft->HasTypedField("Open")) + liveValues.opennessLeftHand = objHandLeft->GetNumberField("Open"); } if (objHandRight != nullptr && objHandRight.IsValid()) { vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandRight(vecHandRight); + if (objHandRight->HasTypedField("Open")) + liveValues.opennessRightHand = objHandRight->GetNumberField("Open"); } - liveValues.ProcessVerboseBody(verbose); - liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); - liveValues.ProcesssVerboseVectorsHandRight(vecHandRight); liveValues.jumpHeight = verbose.Events.Jump.Magnitude; visibilityFlags.ProcessVerbose(verbose.Scalars); } +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { for (int32 i = begin; i < begin + quatArray.Num(); i++) { const FName& jointName = jointNames[i]; @@ -453,7 +454,6 @@ void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& com } void PoseAIRig::CachePose(const TArray& transforms) { - //cachedPose.Empty(); cachedPose = TArray(transforms); } @@ -531,7 +531,6 @@ void PoseAIRigUE4::Configure() } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 170.0f; rig = MakeStaticData(); } @@ -608,10 +607,85 @@ void PoseAIRigMixamo::Configure() AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 161.0f; rig = MakeStaticData(); } +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + + void PoseAIRigMetaHuman::Configure() { @@ -699,13 +773,16 @@ void PoseAIRigMetaHuman::Configure() AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 168.0f; rig = MakeStaticData(); } void PoseAIRigDazUE::Configure() { - + //daz has extra joints in the legs + rShinJoint = 5; + lShinJoint = 10; + lowerBodyNumOfJoints = 11; + jointNames.Empty(); boneVectors.Empty(); parentIndices.Empty(); @@ -793,9 +870,8 @@ void PoseAIRigDazUE::Configure() AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); } numHandJoints = (jointNames.Num() - numBodyJoints) / 2; - rigHeight = 168.0f; rig = MakeStaticData(); } - +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp new file mode 100644 index 0000000..eec8f4a --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -0,0 +1,387 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIStructs.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +// Utility conversion functions for compact representation and from arrays to vectors + +float UintB64ToUint(char a, char b) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + + return reverse_map[static_cast(a)] * 64 + reverse_map[static_cast(b)]; +} +uint32 UintB64ToUint(char a, char b, char c) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + return reverse_map[static_cast(a)] * 4096 + reverse_map[static_cast(b)] * 64 + reverse_map[static_cast(c)]; +} + +float FixedB64pairToFloat(char a, char b) { + static const float firstByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9384465070835368, 0.9697117733268197, 0.9384465070835368, 0.9384465070835368, 0.9697117733268197, 0.6257938446507083, 0.6570591108939912, 0.688324377137274, 0.7195896433805569, 0.7508549096238397, 0.7821201758671226, 0.8133854421104054, 0.8446507083536883, 0.8759159745969711, 0.907181240840254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, -0.9687347337567171, -0.9374694675134343, -0.9062042012701514, -0.8749389350268686, -0.8436736687835857, -0.8124084025403029, -0.78114313629702, -0.7498778700537372, -0.7186126038104543, -0.6873473375671715, -0.6560820713238886, -0.6248168050806058, -0.5935515388373229, -0.5622862725940401, -0.5310210063507572, -0.49975574010747437, -0.4684904738641915, -0.43722520762090866, -0.4059599413776258, -0.37469467513434296, -0.3434294088910601, -0.31216414264777725, -0.2808988764044944, -0.24963361016121155, -0.2183683439179287, 0.0, 0.0, 0.0, 0.0, 0.9697117733268197, 0.0, -0.18710307767464585, -0.155837811431363, -0.12457254518808014, -0.09330727894479729, -0.06204201270151444, -0.030776746458231585, 0.0004885197850512668, 0.03175378602833412, 0.06301905227161697, 0.09428431851489982, 0.12554958475818268, 0.15681485100146553, 0.18808011724474838, 0.21934538348803123, 0.2506106497313141, 0.28187591597459694, 0.3131411822178798, 0.34440644846116264, 0.3756717147044455, 0.40693698094772834, 0.4382022471910112, 0.46946751343429405, 0.5007327796775769, 0.5319980459208598, 0.5632633121641426, 0.5945285784074255 }; + static const float secondByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.030288226673180263, 0.030776746458231558, 0.030288226673180263, 0.030288226673180263, 0.030776746458231558, 0.025403028822667317, 0.025891548607718612, 0.026380068392769906, 0.0268685881778212, 0.027357107962872496, 0.02784562774792379, 0.028334147532975085, 0.02882266731802638, 0.029311187103077674, 0.02979970688812897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004885197850512946, 0.0009770395701025891, 0.0014655593551538837, 0.0019540791402051783, 0.002442598925256473, 0.0029311187103077674, 0.003419638495359062, 0.0039081582804103565, 0.004396678065461651, 0.004885197850512946, 0.00537371763556424, 0.005862237420615535, 0.006350757205666829, 0.006839276990718124, 0.0073277967757694185, 0.007816316560820713, 0.008304836345872008, 0.008793356130923302, 0.009281875915974597, 0.009770395701025891, 0.010258915486077186, 0.01074743527112848, 0.011235955056179775, 0.01172447484123107, 0.012212994626282364, 0.0, 0.0, 0.0, 0.0, 0.030776746458231558, 0.0, 0.012701514411333659, 0.013190034196384953, 0.013678553981436248, 0.014167073766487542, 0.014655593551538837, 0.015144113336590131, 0.015632633121641426, 0.01612115290669272, 0.016609672691744015, 0.01709819247679531, 0.017586712261846604, 0.0180752320468979, 0.018563751831949193, 0.019052271617000488, 0.019540791402051783, 0.020029311187103077, 0.02051783097215437, 0.021006350757205666, 0.02149487054225696, 0.021983390327308255, 0.02247191011235955, 0.022960429897410845, 0.02344894968246214, 0.023937469467513434, 0.024425989252564728, 0.024914509037616023 }; + return firstByte[static_cast(a)] + secondByte[static_cast(b)]; +} + +void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { + flatArray.Reserve(flatArray.Num() + data.Len() / 2); + for (int i = 0; i + 1 < data.Len(); i += 2) + flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); +} + +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { + quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); + for (int i = 0; i + 3 < flatArray.Num(); i += 4) + quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); +} + + +void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ + fieldVector2D.X = (*value)[0]->AsNumber(); + fieldVector2D.Y = (*value)[1]->AsNumber(); + } +} + +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + +void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { + if (value.Num() >= 2) { + fieldVector2D.X = value[0]; + fieldVector2D.Y = value[1]; + } +} + +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + +void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { + if (newValue != field) + changeFlag = true; + field = newValue; +} +void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { + SetAndCheckForChange(newValue > 0.5f, field, changeFlag); +} +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} + +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} + +FString FPoseAIHandshake::ToString() const { + return FString::Printf( + TEXT("{\"HANDSHAKE\":{" + "\"name\":\"Unreal LiveLink\"," + "\"rig\":\"%s\", " + "\"mode\":\"%s\", " + "\"face\":\"%s\", " + "\"context\":\"%s\", " + "\"whoami\":\"%s\", " + "\"signature\":\"%s\", " + "\"mirror\":\"%s\", " + "\"syncFPS\": %d, " + "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " + "\"packetFormat\": %d" + "}}"), + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), + *whoami, + *signature, + *(YesNoString(isMirrored)), + syncFPS, + cameraFPS, + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) + ); +} + + +bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const +{ + return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; +} + + + +FString FPoseAIModelConfig::ToString() const { + return FString::Printf( + TEXT("{\"CONFIG\":{" + "\"mirror\":\"%s\", " + "\"stepSensitivity\":%f, " + "\"armSensitivity\":%f, " + "\"crouchSensitivity\": %f, " + "\"jumpSensitivity\":%f" + "}}"), + *(YesNoString(isMirrored)), + stepSensitivity, + armSensitivity, + crouchSensitivity, + jumpSensitivity + ); +} + +bool FPoseAIEventPairBase::CheckTriggerAndUpdate() { + bool hasChanged = Count != InternalCount; + InternalCount = Count; + return hasChanged; +} + +void FPoseAIEventPair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Magnitude = FixedB64pairToFloat(compactString[3], compactString[4]); +} + +void FPoseAIGesturePair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Current = UintB64ToUint(compactString[3], compactString[4]); +} + +void FPoseAIEventStruct::ProcessCompactBody(const FString& compactString) { + TArray compactOrder = { &Footstep, &SidestepL, &SidestepR, &Jump, &FeetSplit, &ArmPump, &ArmFlex, &ArmGestureL, &ArmGestureR}; + if (compactString.Len() % 5 != 0) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: Invalid event string: %s."), *compactString); + return; + } + for (int i = 0; i < compactOrder.Num(); ++i) { + if (compactString.Len() < 5 * i + 5) + break; + compactOrder[i]->ProcessCompact(compactString.Mid(i * 5, 5)); + } +} + +void FPoseAIVisibilityFlags::ProcessCompact(const FString& visString) { + hasChanged = false; + SetAndCheckForChange(visString[0] != '0', isTorso, hasChanged); + SetAndCheckForChange(visString[1] != '0', isLeftLeg, hasChanged); + SetAndCheckForChange(visString[2] != '0', isRightLeg, hasChanged); + SetAndCheckForChange(visString[3] != '0', isLeftArm, hasChanged); + SetAndCheckForChange(visString[4] != '0', isRightArm, hasChanged); + if (visString.Len() > 5) + SetAndCheckForChange(visString[5] != '0', isFace, hasChanged); +} + +void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) { + int32 idx = 0; + if(compactString.Len() < 14) return; + bodyHeight = FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) + 1.0f; + chestYaw = FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f; + stanceYaw = FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 180.0f; + stableFeet = UintB64ToUint(compactString[idx + 6], compactString[idx + 7]); + handZoneLeft = UintB64ToUint(compactString[idx + 8], compactString[idx + 9]); + handZoneRight = UintB64ToUint(compactString[idx + 10], compactString[idx + 11]); + isCrouching = UintB64ToUint(compactString[idx + 12], compactString[idx + 13]) > 0; +} + +void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { + //tbd - this could be simplified if we don't need to keep supported older versions of the api + int32 idx = 0; + if (compactString.Len() < 12) return; + upperBodyLean.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 180.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f + ); + idx += 4; + hipScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx+1]), + FixedB64pairToFloat(compactString[idx+2], compactString[idx+3]) + ); + idx += 4; + chestScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]), + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) + ); + idx += 4; + if (compactString.Len() < idx + 12) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + handIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + if (compactString.Len() < idx + 18) return; + rootTranslation.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; +} + +void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessLeftHand = handObj->GetNumberField("Open"); +} + +void FPoseAILiveValues::ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessRightHand = handObj->GetNumberField("Open"); +} + + +void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { + hasChanged = false; + SetAndCheckForChange(scalars.VisTorso, isTorso, hasChanged); + SetAndCheckForChange(scalars.VisLegL, isLeftLeg, hasChanged); + SetAndCheckForChange(scalars.VisLegR, isRightLeg, hasChanged); + SetAndCheckForChange(scalars.VisArmL, isLeftArm, hasChanged); + SetAndCheckForChange(scalars.VisArmR, isRightArm, hasChanged); +} + + +const FString FPoseAILiveValues::fieldPointScreen = FString(TEXT("PointScreen")); +const FString FPoseAILiveValues::fieldThumbScreen = FString(TEXT("ThumbScreen")); + +void FPoseAIScalarStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody) { + FJsonObjectConverter::JsonObjectToUStruct(scaBody.ToSharedRef(), this); +} + +void FPoseAIEventStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody) { + FJsonObjectConverter::JsonObjectToUStruct(eveBody.ToSharedRef(), this); +} + +void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) { + FJsonObjectConverter::JsonObjectToUStruct(jsonObj.ToSharedRef(), this); +} + + +void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ + bodyHeight = verbose.Scalars.BodyHeight; + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); + stanceYaw = verbose.Scalars.StanceYaw * 180.0f; + chestYaw = verbose.Scalars.ChestYaw * 180.0f; + isCrouching = verbose.Scalars.IsCrouching > 0.5f; + handZoneLeft = verbose.Scalars.HandZoneL; + handZoneRight= verbose.Scalars.HandZoneR; + ProcessArrayAsVector2D(verbose.Vectors.HipLean, upperBodyLean); + upperBodyLean *= 180.0f; + ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); + ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); + ProcessArrayAsVector3D(verbose.Vectors.Hip, rootTranslation); + ProcessArrayAsVector3D(verbose.Vectors.FootIkL, footIkL); + ProcessArrayAsVector3D(verbose.Vectors.FootIkR, footIkR); +} + + + +void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); +} + +void FPoseAILiveValues::ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); +} + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp new file mode 100644 index 0000000..70554f9 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -0,0 +1,291 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "SPoseAILiveLinkWidget.h" +#include "PoseAILiveLinkSourceFactory.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +TWeakPtr SPoseAILiveLinkWidget::source = nullptr; + +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking +static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; + +const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; +int32 SPoseAILiveLinkWidget::syncFPS = 60; +int32 SPoseAILiveLinkWidget::cameraFPS = 60; +int32 SPoseAILiveLinkWidget::modeIndex = 0; +int32 SPoseAILiveLinkWidget::rigIndex = 0; +bool SPoseAILiveLinkWidget::isMirrored = false; +bool SPoseAILiveLinkWidget::isIPv6 = false; + +BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION +void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) +{ + GConfig->GetBool(*section, TEXT("isIPv6"), isIPv6, GEditorIni); + GConfig->GetBool(*section, TEXT("isMirrored"), isMirrored, GEditorIni); + + GConfig->GetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + if (modeIndex < 0 || modeIndex >= PoseAI_Modes.Num()) + modeIndex = 0; + GConfig->GetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + if (rigIndex < 0 || rigIndex >= PoseAI_Rigs.Num()) + rigIndex = 0; + + GConfig->GetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + + ChildSlot + [ + SNew(SBox).WidthOverride(300) + [ + SNew(SVerticalBox) + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("UseIPv6", "Connect via an IPv6 socket")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(ipv6CheckBox, SCheckBox) + .IsChecked(isIPv6 ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.75f) + [ + SNew(STextBlock).Text(LOCTEXT("PortIPv4", "Port for IPv4 (0 if unused)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).FillWidth(0.25f) + [ + SAssignNew(portInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdatePort) + .Text(FText::FromString(FString::FromInt(portNum))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.65f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleMode", "Toggle Camera Mode")) + + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.45f) + + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleModeClicked) + [ + SAssignNew(modeInput, STextBlock) + .Text(FText::FromString(PoseAI_Modes[modeIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.6f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleRig", "Toggle rig format")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.4f) + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleRigClicked) + [ + SAssignNew(rigInput, STextBlock) + .Text(FText::FromString(PoseAI_Rigs[rigIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("IsMirrored", "Mirror camera (flip left/right)")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(mirroredCheckBox, SCheckBox) + .IsChecked(isMirrored ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("SyncFPS", "Smoothed FPS (by app)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(syncFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateSyncFPS) + .Text(FText::FromString(FString::FromInt(syncFPS))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("CameraFPS", "Request Camera FPS")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(cameraFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateCameraFPS) + .Text(FText::FromString(FString::FromInt(cameraFPS))) + ] + ] + + SVerticalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).AutoHeight() + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnOkClicked) + [ + SNew(STextBlock) + .Text(LOCTEXT("OK", "OK")) + ] + ] + ] + ]; +} +END_SLATE_FUNCTION_BUILD_OPTIMIZATION + + +void SPoseAILiveLinkWidget::UpdatePort(const FText& InText, ETextCommit::Type type) +{ + portNum = FCString::Atoi(*(InText.ToString())); +} + + +bool SPoseAILiveLinkWidget::IsPortValid() const +{ + if (portNum < 1028 || portNum > 49151) { + FLiveLinkLog::Warning(TEXT("PoseAI: %d is an invalid port number. Set a valid port number (>1028 and <49151)."), portNum); + return false; + } + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); + return false; + } + return true; +} + + +void SPoseAILiveLinkWidget::UpdateSyncFPS(const FText& InText, ETextCommit::Type type) +{ + syncFPS = FCString::Atoi(*(InText.ToString())); + if (syncFPS < 0) { + syncFPS = 0; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + +void SPoseAILiveLinkWidget::UpdateCameraFPS(const FText& InText, ETextCommit::Type type) +{ + cameraFPS = FCString::Atoi(*(InText.ToString())); + if (cameraFPS < 24) { + cameraFPS = 24; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + GConfig->SetInt(*section, TEXT("cameraFPS"), cameraFPS, GEditorIni); + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + + +void SPoseAILiveLinkWidget::disableExistingSource() +{ + TSharedPtr linkSource = source.Pin(); + if (linkSource.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); + } +} +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() +{ + FPoseAIHandshake handshake = FPoseAIHandshake(); + handshake.isMirrored = isMirrored; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); + handshake.syncFPS = syncFPS; + handshake.cameraFPS = cameraFPS; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); +} + +FReply SPoseAILiveLinkWidget::OnToggleModeClicked() +{ + modeIndex = (modeIndex+1) % PoseAI_Modes.Num(); + modeInput->SetText(FText::FromString(PoseAI_Modes[modeIndex])); + GConfig->SetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnToggleRigClicked() +{ + rigIndex = (rigIndex + 1) % PoseAI_Rigs.Num(); + rigInput->SetText(FText::FromString(PoseAI_Rigs[rigIndex])); + GConfig->SetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnOkClicked() +{ + ReadCheckBox(mirroredCheckBox, isMirrored); + ReadCheckBox(ipv6CheckBox, isIPv6); + + GConfig->SetBool(*section, TEXT("isMirror"), isMirrored, GEditorIni); + + if (IsPortValid()) { + GConfig->SetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + GConfig->Flush(false, GEditorIni); + FString connectionString = ""; + TSharedPtr src = CreateSource(connectionString); + callback.Execute(src, connectionString); + FLiveLinkLog::Info( + TEXT("PoseAI: Setup source. Rig is in %s format, %s and %s."), + *PoseAI_Rigs[rigIndex], + *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")) + ); + } + return FReply::Handled(); +} + +void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& readTo) +{ + TSharedPtr pin = checkBox.Pin(); + if (pin) + readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h new file mode 100644 index 0000000..977dea1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -0,0 +1,128 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "IPAddress.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "SocketSubsystem.h" + + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port); + + +/** + * Implements a more generic endpoint to allow for both IPv6 and IPv4 networks, based on Epic Games IPv4 endpoint from the Networking module. + * + * Mainly is a wrapper around an FInternetAddr with the helper functions needed to use the networking code with only minor modifications. + * + */ +struct FPoseAIEndpoint +{ + /** Holds the endpoint's IP address. */ + TSharedPtr Address; + + /** Holds the endpoint's port number. */ + uint16 Port; + +public: + + /** Default constructor. */ + FPoseAIEndpoint() { } + + + /** + * Creates and initializes a new endpoint from a given FInternetAddr object. + * + * Note: this constructor will be removed after the socket subsystem has been refactored. + * + * @param InternetAddr The Internet address. + */ + + + FPoseAIEndpoint(const TSharedPtr& InternetAddr) + { + check(InternetAddr.IsValid()); + + int32 OutPort; + Address = InternetAddr; + InternetAddr->GetPort(OutPort); + Port = OutPort; + } + + bool IsValid() const { return Address != nullptr && Address.IsValid(); } + +public: + + /** + * Compares this endpoint with the given endpoint for equality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are equal, false otherwise. + */ + bool operator==(const FPoseAIEndpoint& Other) const + { + return ((Address == Other.Address)); + } + + /** + * Compares this address with the given endpoint for inequality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are not equal, false otherwise. + */ + bool operator!=(const FPoseAIEndpoint& Other) const + { + return (Address != Other.Address); + } + + +public: + + + /** + * Gets a string representation for this endpoint. + * + * @return String representation. + * @see Parse, ToText + */ + POSEAILIVELINK_API FString ToString() const; + + /** + * Gets the display text representation. + * + * @return Text representation. + * @see ToString + */ + FText ToText() const + { + return FText::FromString(ToString()); + } + + TSharedRef ToInternetAddr() const + { + if (CachedSocketSubsystem == nullptr) + Initialize(); + + check(CachedSocketSubsystem != nullptr && "Networking module not loaded and initialized"); + TSharedRef InternetAddr = CachedSocketSubsystem->CreateInternetAddr(Address->GetProtocolType()); + { + InternetAddr->SetRawIp(Address->GetRawIp()); + InternetAddr->SetPort(Address->GetPort()); + } + + return InternetAddr; + } + +public: + + /** Initializes the IP endpoint functionality. */ + static void Initialize(); + + + +private: + /** ISocketSubsystem::Get() is not thread-safe, so we cache it here. */ + static ISocketSubsystem* CachedSocketSubsystem; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h new file mode 100644 index 0000000..3eecfa0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -0,0 +1,395 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "Async/Async.h" +#include "LiveLinkTypes.h" +#include "PoseAIStructs.h" +#include "PoseAIEventDispatcher.generated.h" + + +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); +DECLARE_MULTICAST_DELEGATE_TwoParams(FPoseAIConfigUpdate, const FLiveLinkSubjectName&, FPoseAIModelConfig); + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAISubjectConnected, const FLiveLinkSubjectName&, SubjectName, bool, isReconnection); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIRegisteredAs, const FLiveLinkSubjectName&, SubjectName, FName, ConnectionName); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIFrameReceived); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIVisibilityChange, const FPoseAIVisibilityFlags&, Flags); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAILiveValuesUpdate, const FPoseAILiveValues&, LiveValues); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootstepEvent, float, height, bool, isLeftFoot); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAISidestepEvent, bool, isLeftStep); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootsplitEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmpumpEvent, float, height); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIArmflexEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmjackEvent,bool, isRising); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIArmflapEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIJumpEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAICrouchEvent, bool, isCrouching); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIHandToZoneEvent, int32, zone); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmGestureEvent, int32, armGesture); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIStationaryEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIResetLivePositionEvent); + + +UCLASS(ClassGroup = (PoseAI)) +class POSEAILIVELINK_API UStepCounter : public UObject +{ + GENERATED_BODY() + +public: + /** Convenience setter for timeout and fadeduration at the same time*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + UStepCounter* SetProperties(float timeoutIn = 0.5f, float fadeDuration = 0.2f); + + /** clears all steps. Optionally fades speed over */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + void Halt(bool fade); + + /** Average step distance in [body width, body height] units per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float DistancePerSecond(); + + /** Average number of steps per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float StepsPerSecond(); + + /** Time since last step in seconds*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float TimeSinceLastStep(); + + /** most recent step distance*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float LastDistance(); + + + /** time since last step when motion begins to slow*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float timeout = 0.5f; + + /** after a next step timeout, the speed is faded to zero over this duration*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float fadeDurationOnTimeout = 0.2f; + + /** total steps registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + int32 totalSteps = 0; + + /** total distance registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + float totalDistance = 0.0f; + + void RegisterStep(float stepDistance); + + UStepCounter(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) + { + + times_.SetNumUninitialized(num_to_track); + heights_.SetNumUninitialized(num_to_track); + } + + +private: + float CheckIfActiveAndFade(); + const int32 num_to_track = 4; + int32 num_ = 0; + int32 tail_ = -1; + + float steps_per_second_ = 0.0f; + float distance_per_second_ = 0.0f; + FDateTime last_time_ = FDateTime::Now(); + TArray times_; + TArray heights_; +}; + + +class UPoseAIEventDispatcher; + +UCLASS(ClassGroup = (PoseAI), meta = (BlueprintSpawnableComponent)) +class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent +{ + GENERATED_BODY() + friend UPoseAIEventDispatcher; + + public: + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple portss (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum=8080, bool isIPv6 = false); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + + /** Sends a message to the connected PoseCamera to reconfigure the model with user settings */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void ChangeModelConfig(FPoseAIModelConfig config); + + /** sends disconnect message to app and closes source, freeing up Port*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(); + + /** sends disconnect message to app but does not clsoe source */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Disconnect(); + + /** Get the LiveLink subject name associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectName() { return subjectName; } + + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void RegisterAsFirstAvailable(); + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + static void SetHandshake(const FPoseAIHandshake& handshake); + + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken = true); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Deregister(); + + /** Sets rig height, which scales root motion, and allows scaling per dimension (i.e. set Y=0 for no motion to/from camera) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ScaleMotion(float RigHeight=170.0f, FVector Scale=FVector(1.0f, 1.0f,1.0f)); + + /** compensates for an active source's camera rotation in the real world */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void SetLiveCameraRotation(float pitch, float yaw = 0.0f, float roll=0.0f); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseAsBaseTranslation(); + + /** Have player stand up straight and face screen as part of configuration */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseToOrientCamera(); + + /** Remove all live root motion (sets scalemotion to zero)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ZeroMotion(); + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FDateTime lastFrameReceived; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAIVisibilityFlags visibilityFlags; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAILiveValues mostRecentValues; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* footsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* leftsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* rightsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* feetsplits; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armpumps ; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflexes; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armjacks; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapL; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapR; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* jumps; + + /********** events ********************/ + + /** when the component succeesfully registered with a connection (i.e. a pose camera joined) */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIRegisteredAs onRegistered; + + /** when any body part visisbility flag changes */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIVisibilityChange onVisibilityChange; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAILiveValuesUpdate onLiveValues; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootstepEvent onFootstep; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootsplitEvent onFeetsplit; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepLeftFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepRightFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmpumpEvent onArmpump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflexEvent onArmflex; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmjackEvent onArmjack; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureLeft; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureRight; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIJumpEvent onJump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAICrouchEvent onCrouch; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIStationaryEvent onStationary; + + void SetLiveValues(FPoseAILiveValues values) { + mostRecentValues = values; + } + + virtual void InitializeComponent() override { + Super::InitializeComponent(); + InitializeObjects(); + } + +private: + FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + + void InitializeObjects(); + +}; + + +/** + * Dispatcher to transmit events to blueprints and c++, can access events from all sources + */ +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAIEventDispatcher : public UObject +{ +GENERATED_BODY() +public: + /** Gets the singleton dispatcher. Use this for binding or to get named components */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + static UPoseAIEventDispatcher* GetDispatcher() { + if (theInstance==nullptr) { + theInstance = NewObject(); + theInstance->AddToRoot(); + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: Creating EventDispatcher.")); + } + return theInstance; + } + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void SetHandshake(const FPoseAIHandshake& handshake); + + FPoseAIHandshakeUpdate handshakeUpdate; + FPoseAIConfigUpdate modelConfigUpdate; + FPoseAIDisconnect disconnect; + FPoseAIDisconnect closeSource; + + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + + /** Convenience event to tell animation blueprin*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + void BroadcastResetLivePosition(); + + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISubjectConnected subjectConnected; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIResetLivePositionEvent resetLivePositionEvent; + + // Connection driven events + void BroadcastCloseSource(const FLiveLinkSubjectName& subjectName); + void BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config); + void BroadcastDisconnect(const FLiveLinkSubjectName& subjectName); + void BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName); + void BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName); + + // Pose Camera driven events + void BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight); + void BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising); + void BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching); + void BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep); + void BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastJumps(const FLiveLinkSubjectName& subjectName); + void BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values); + void BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastStationary(const FLiveLinkSubjectName& subjectName); + void BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags); + + bool RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken); + void RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component); + + bool HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component); + + void BroadcastResetZeroLivePosition(); + +private: + static UPoseAIEventDispatcher* theInstance; + const double timeoutInSeconds = 60.0; + TQueue componentQueue; + UPROPERTY() + TMap componentsByName; + TMap knownConnectionsWithTime; + UPoseAIEventDispatcher() : UObject() {}; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h new file mode 100644 index 0000000..75d849b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..13d5dee --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,105 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..56ac13d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,72 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h similarity index 50% rename from UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h rename to UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h index 77193e5..6fd81a0 100644 --- a/UnrealEngineAPI/PluginV1.3/5.0/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSingleSource.h +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -1,4 +1,4 @@ -// Copyright Pose AI Ltd 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. #pragma once @@ -19,9 +19,8 @@ #include "PoseAIRig.h" #include "PoseAILiveLinkServer.h" #include "PoseAIStructs.h" -#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkFaceSubSource.h" -#define LOCTEXT_NAMESPACE "PoseAI" struct POSEAILIVELINK_API PoseAIPortRecord { @@ -30,70 +29,116 @@ struct POSEAILIVELINK_API PoseAIPortRecord { FLiveLinkSubjectKey subjectKey; }; +class PoseAILiveLinkSingleSourceListener; + + /** * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to * trigger frame events and update the LiveLink pose source information. */ -class POSEAILIVELINK_API PoseAILiveLinkSingleSource : public ILiveLinkSource +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource { public: - static const int32 portDefault = 8080; - PoseAILiveLinkSingleSource(int32 port, bool useIPv6, const FPoseAIHandshake& handshake); + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); // standard Live Link source methods virtual bool CanBeDisplayedInUI() const { return true; } virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } - virtual FText GetSourceType() const { - return LOCTEXT("SourceType", "PoseAI mobile"); - } - virtual FText GetSourceMachineName() const { - return LOCTEXT("SourceMachineName", "Unreal Engine");; - } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; virtual FText GetSourceStatus() const { return status; } - virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) {} + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; virtual bool IsSourceStillValid() const override; virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} - virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid); + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; virtual bool RequestSourceShutdown(); - virtual void Update() override; - - - // method to add a source from code, instead of relying on presets. Exposed to blueprints via the PoseAI movement component. - static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + virtual void Update() override {} + // custom methods static bool GetPortGuid(int32 port, FGuid& fguid); static bool IsValidPort(int32 port); - - //Assigns a Livelink subject name from a port number. Returns FName as that class is used in LiveLinkSubjectKey consturctor - static FName SubjectNameFromPort(int32 port); - - //Looks up the "connection name" associated with port/subjetname which is the phone name @ IP Address. static FName GetConnectionName(int32 port); static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); - void SetConnectionName(FName name); - + static FName SubjectNameFromPort(int32 port); + void disable(); FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } - void UpdatePose(TSharedPtr rig, TSharedPtr jsonPose); + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; int32 port; FGuid sourceGuid ; FLiveLinkSubjectKey subjectKey; - TSharedPtr udpServer; - - /* stores ports across different sources to avoid conflict from user input */ - static TMap usedPorts; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; - ILiveLinkClient* liveLinkClient = nullptr; - ILiveLinkClient* client = nullptr; - - UPoseAIEventDispatcher* dispatcher; + void AddSubject(); - mutable FText status; - - void AddSubject(TSharedPtr rig); +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + }; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h new file mode 100644 index 0000000..1819982 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -0,0 +1,37 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once +#include "CoreMinimal.h" +#include "LiveLinkRetargetAsset.h" +#include "PoseAILiveLinkRetargetRotations.generated.h" + +// Rretarget asset for data coming from Live Link. Remaps rotations onto all bones and only translations for root and pelvis/hip. +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAILiveLinkRetargetRotations : public ULiveLinkRetargetAsset +{ + GENERATED_UCLASS_BODY() + + virtual ~UPoseAILiveLinkRetargetRotations() {} + + //~ Begin UObject Interface + virtual void BeginDestroy() override; + //~ End UObject Interface + + //~ Begin ULiveLinkRetargetAsset interface + virtual void BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) override; + //~ End ULiveLinkRetargetAsset interface + + // allow user to scale translations for differences in skeleton sizes + UPROPERTY(EditAnywhere, Category = Settings) + float scaleTranslation = 1.0f; + +private: + + void OnBlueprintClassCompiled(UBlueprint* TargetBlueprint); + + +#if WITH_EDITOR + /** Blueprint.OnCompiled delegate handle */ + FDelegateHandle OnBlueprintCompiledDelegate; +#endif +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h new file mode 100644 index 0000000..c6d53c7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -0,0 +1,217 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Runtime/Networking/Public/Networking.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "Runtime/Sockets/Public/SocketSubsystem.h" +#include "HAL/RunnableThread.h" +#include "LiveLinkLog.h" +#include "SocketTypes.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "IPAddress.h" +#include "Json.h" +#include "PoseAIStructs.h" +#include "PoseAIUdpSocketReceiver.h" +#include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" + + +class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; +class FPoseAISocketSender; + +// The networking class needs to be rewritten + +class POSEAILIVELINK_API PoseAILiveLinkServer +{ +public: + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); + + ~PoseAILiveLinkServer() { + CleanUp(); + } + + // utility function that identifies host IPv4 address, to be printed in LiveLink console to help user connect to correct address + static bool GetIP(FString& myIP); + + void CleanUp(); + void Disconnect(); + + TSharedPtr GetSocket() const { return serverSocket; } + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; + void SendHandshake() const; + void SetHandshake(const FPoseAIHandshake& handshake); + // receiver will be set on a runnable thread and set once started + void SetReceiver(TSharedPtr receiver) { udpSocketReceiver = receiver; } + + +private: + const static FString fieldPrettyName; + const static FString fieldVersion; + const static FString fieldUUID; + const static FString fieldRigType; + const static FString requiredMinVersion; + + TSharedPtr listener; + TWeakPtr source_; + FPoseAIHandshake handshake; + FName protocolType; + int32 port; + bool cleaningUp = false; + + // time of last connection. After timeout seconds a newer connection can takeover the port. + FDateTime lastConnection; + const double TIMEOUT_SECONDS = 10.0; + + TSharedPtr serverSocket; + + //used to launch receiver without slowing main thread + TSharedPtr poseAILiveLinkRunnable; + + //Listens for packets + TSharedPtr udpSocketReceiver; + + //sends instructions to paired app + TSharedPtr udpSocketSender; + FPoseAIEndpoint endpoint; + + // disconnect message formatted for Pose AI mobile app + FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); + + void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + + + bool HasValidConnection() const; + + FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; + + // make sure mobile app is sufficiently advanced version as both endpoints of software evolve + bool CheckAppVersion(FString version) const; + + //split clean up routine by component + void CleanUpReceiver(); + void CleanUpSender(); + void CleanUpSocket(); + +}; + + + +class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable +{ +public: + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { + myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); + thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); + } + virtual uint32 Run() override; + +protected: + FString myName; + int32 port; + FRunnableThread* thread = nullptr; +private: + PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; + TSharedPtr udpSocketReceiver; +}; + + + + +// built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update +class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable +{ +public: + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { + thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); + } + + + virtual uint32 Run() override { + while (running && thread == nullptr) { + FPlatformProcess::Sleep(0.2); + } + + while (running ) { + Sleep(true); + while (running && sleeping) { + FPlatformProcess::Sleep(0.005); + } + + } + + thread = nullptr; + return 0; + } + + virtual void Stop() override { + running = false; + if (thread != nullptr) { + Sleep(false); + } + } + + bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) + { + if (running) { + + int32 sent = 0; + if (!Socket) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); + return false; + } + + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); + + if (sent != Data->Num()) + return false; + + Sleep(false); + return true; + } + return false; + } + + void Sleep(bool sleep) { + sleeping = sleep; + if (thread != nullptr) + thread->Suspend(sleep); + } + +protected: + /** The network socket. */ + TSharedPtr Socket; + + /** The thread object. */ + FRunnableThread* thread = nullptr; + + bool running = true; + bool sleeping = false; +}; + + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h new file mode 100644 index 0000000..73f462b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -0,0 +1,30 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "LiveLinkSourceFactory.h" +#include "ILiveLinkSource.h" +#include "PoseAILiveLinkSourceFactory.generated.h" + +/** + * + */ +UCLASS() +class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFactory +{ + GENERATED_BODY() + + UPoseAILiveLinkSourceFactory() { + } + + ~UPoseAILiveLinkSourceFactory () { + } + +public: + virtual TSharedPtr< SWidget > BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const override; + virtual TSharedPtr< ILiveLinkSource > CreateSource(const FString& ConnectionString) const override; + virtual FText GetSourceDisplayName() const override; + virtual FText GetSourceTooltip() const override; + virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h new file mode 100644 index 0000000..1e8e6ac --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -0,0 +1,151 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "ILiveLinkSource.h" +#include "LiveLinkSubjectSettings.h" +#include "Roles/LiveLinkAnimationTypes.h" +#include "Json.h" +#include "PoseAIStructs.h" + +struct POSEAILIVELINK_API Remapping +{ + FName TargetJointName; + FQuat RotAdj; + Remapping(FName TargetJointName, FQuat RotAdj) : TargetJointName(TargetJointName), RotAdj(RotAdj) {}; +}; + + + +/** + * Abstract base class for the different rig formats streamable by Pose AI + */ +class POSEAILIVELINK_API PoseAIRig +{ + public: + FLiveLinkStaticDataStruct MakeStaticData(); + bool ProcessFrame(const TSharedPtr, FLiveLinkAnimationFrameData& data); + static bool IsFrameData(const TSharedPtr jsonObject); + static TSharedPtr PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake); + static TWeakPtr GetRigFromSubjectName(const FLiveLinkSubjectName& name); + FName RigType() { return rigType; } + + FPoseAIVisibilityFlags visibilityFlags; + FPoseAILiveValues liveValues; + FPoseAIScalarStruct scalars; + FPoseAIEventStruct events; + + bool useNextRootAsOffset = false; + //ankle to head top height for scaling PoseAI root motion. + float rigHeight = 170.0f; + + float CameraTilt = 0.0f; + + protected: + FLiveLinkStaticDataStruct rig; + FPoseAIVerbose verbose; + static const FString fieldBody; + static const FString fieldRigType; + static const FString fieldHandLeft; + static const FString fieldHandRight; + static const FString fieldRotations; + static const FString fieldEvents; + static const FString fieldScalars; + static const FString fieldVectors; + + protected: + PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake); + virtual ~PoseAIRig() { + }; + + /* sets up skeletal heirarchy and provides default locations for each transform based on default skeleton (i.e. UE4 Mannequen or male Metahuman). + These bone lengths ensure a sensible animation is created even if user does not retarget from livelink */ + virtual void Configure(); //impure for MacOS compatibility + + FLiveLinkSubjectName name; + FName rigType; + + bool includeHands; + bool isMirrored; + bool isLowerBodyRotated; + bool isDesktop; + int32 numBodyJoints = 21; + int32 numHandJoints = 17; + // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) + int32 lowerBodyNumOfJoints = 8; + + int32 rShinJoint = 3; + int32 lShinJoint = 7; + + bool isCrouching = false; + int32 handZoneL = 5; + int32 handZoneR = 5; + int32 stableFeet = 0; + FVector prevRootTranslation = FVector::ZeroVector; + // store translations of deployed rig + TMap boneVectors; + TArray jointNames; + TArray parentIndices; + TArray cachedPose = {}; + + //temporary variable used for convenience in rig construction + FName lastBoneAdded; + + //extra offset for hip bone to accomodate mesh thickness from bone sockets. + float rootHipOffsetZ = 2.0f; + + void AddBone(FName boneName, FName parentName, FVector translation); + void AddBoneToLast(FName boneName, FVector translation); + void CachePose(const TArray& transforms); + void AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + bool ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + bool ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void TriggerEvents(); + void RotateLowerBody180(TArray& quatArray); + + +private: + static TMap> RigMap; +}; + +class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { + public: + PoseAIRigUE4(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { + public: + PoseAIRigMixamo(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + +class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { +public: + PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { +public: + PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h new file mode 100644 index 0000000..32ae5d2 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -0,0 +1,529 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Json.h" +#include "JsonObjectConverter.h" +#include "PoseAIStructs.generated.h" + + + +/* decoding utilities for compact representation */ +float UintB64ToUint(char a, char b); +uint32 UintB64ToUint(char a, char b, char c); +float FixedB64pairToFloat(char a, char b); +void FStringFixed12ToFloat(const FString& data, TArray& flatArray); +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); + +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + + +/* the handshake configures the main parameters of pose camera*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIHandshake +{ + GENERATED_BODY() + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiAppModes mode = EPoseAiAppModes::Room; + + /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; + + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; + + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; + + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString whoami = ""; + + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString signature = ""; + + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + + + bool operator==(const FPoseAIHandshake& Other) const; + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + + +/*adjusts the sensitivity of PoseAI events*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIModelConfig +{ + GENERATED_BODY() + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float stepSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float armSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float crouchSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float jumpSensitivity = 0.5f; + + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + bool isMirrored = true; + + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + +/** base class for the two event notifications formats sent by camera. All events have a uint count, which upon change signifies a new event has been registered + and a second property, either a float or uint. +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /** number of events registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Count = 0; + + virtual void ProcessCompact(const FString& compactString) {}; + bool CheckTriggerAndUpdate(); +private: + uint32 InternalCount = 0; +}; + +/** +*structure to hold a type of event notification +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**magnitude of event where approrpiate */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + float Magnitude = 0.0f; + + void ProcessCompact(const FString& compactString) override; + +}; + + +USTRUCT() +struct POSEAILIVELINK_API FPoseAIGesturePair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**index code for most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Current = 0; + + void ProcessCompact(const FString& compactString) override; + +}; + + +/** +*structure to receive event notifications +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventStruct +{ + GENERATED_BODY() +public: + /** number of footsteps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Footstep; + + /** number of left foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepL; + + /** number of right foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepR; + + /** number of jumps registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Jump; + + /** number of footsplits registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair FeetSplit; + + /** number of arm pumps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmPump; + + /** number of arm flexes registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmFlex; + + /** number of left or dual arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureL; + + /** number of right arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureR; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody); + void ProcessCompactBody(const FString& compactString); + +}; + +/** +*structure to share additional information +*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIScalarStruct +{ + GENERATED_BODY() +public: + /** visibility flags by body part. Correspond to figure in the app */ + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisTorso = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisFace = 0.0f; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 HandZoneL = 5; + + /** location of right hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PosPeAI") + int32 HandZoneR = 5; + + /** Heading in degrees of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float ChestYaw = 0.0f; + + /** Heading in degrees of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float StanceYaw = 0.0f; + + /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float BodyHeight = 0.0f; + + /** whether subject is crouching */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float IsCrouching = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 StableFoot = 0.0f; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody); + +}; + + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors +{ + GENERATED_BODY() +public: + UPROPERTY() + TArray HipLean; + UPROPERTY() + TArray HipScreen; + UPROPERTY() + TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; + UPROPERTY() + TArray Hip; + UPROPERTY() + TArray FootIkL; + UPROPERTY() + TArray FootIkR; +}; + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerbose +{ + GENERATED_BODY() +public: + UPROPERTY() + FPoseAIEventStruct Events; + UPROPERTY() + FPoseAIScalarStruct Scalars; + UPROPERTY() + FPoseAIVerboseBodyVectors Vectors; + void ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj); + +}; + +/** + * structure to store and expose visibility flags for events alerting programmer if subject is out of camera + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVisibilityFlags +{ + GENERATED_BODY() +public: + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isTorso = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isFace = false; + + bool HasChanged() { return hasChanged; } + void ProcessVerbose(FPoseAIScalarStruct& scalars); + void ProcessCompact(const FString& visString); + +private: + bool hasChanged = false; +}; + +/** + * structure to share additional information + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAILiveValues +{ +GENERATED_BODY() +public: + + /** How much subject is leaning in radians, head-to-hips; x to the side, y forward */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D upperBodyLean = FVector2D(0.0f, 0.0f); + + /** estimated stance height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + bool isCrouching = false; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneLeft = 5; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneRight = 5; + + /** location of subject in camera frame, scaled in body units (i.e. multiply by rig height to translate to game world). Pos Y moves toward camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector rootTranslation = FVector(0.0f, -2.0f, 0.0f); + + /** Heading in radians of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float stanceYaw = 0.0f; + + /** Heading in radians of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float chestYaw = 0.0f; + + /** Heading of flattened left foot to right foot vector relative to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 modelLatency = 0.0f; + + /** timestamp according to pose camera device (CMTime), in seconds */ + double timestamp = 0.0; + + /** DEPR current height of jump in body units */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI_DEPR") + float jumpHeight = 0.0f; + + /** DEPR estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI_DEPR") + float bodyHeight = 0.0f; + + /** location of hips in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D hipScreen = FVector2D(0.0f, 0.0f); + + /** location of chest in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D chestScreen = FVector2D(0.0f, 0.0f); + + /** location of left index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandLeft = FVector2D(0.0f, 0.0f); + + /** location of right index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + + /** location of left thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbLeft = FVector2D(0.0f, 0.0f); + + /** location of right thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbRight = FVector2D(0.0f, 0.0f); + + /** how open the left hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessLeftHand = 0.5f; + + /** how open the right hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessRightHand = 0.5f; + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); + + /** if at least one foot has been stationary for a few frames */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 stableFeet = 0; + + FVector rootOffset = FVector(0.0f, 0.0f, 0.0f); + FRotator cameraRotation = FRotator(0.0f, 0.0f, 0.0f); + FVector scaleMotion = FVector(1.0f, 0.0f, 1.0f); + + void ProcessVerboseBody(const FPoseAIVerbose& scalars); + void ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand); + void ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); + void ProcessCompactScalarsBody(const FString& compactString); + void ProcessCompactVectorsBody(const FString& compactString); + void ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject >); + void ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject >); + +private: + static const FString fieldPointScreen; + static const FString fieldThumbScreen; + +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h similarity index 86% rename from UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h rename to UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h index df42fe5..9434a67 100644 --- a/UnrealEngineAPI/PluginV1.3/4.25/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -1,4 +1,4 @@ -// Pose AI Ltd Copyright 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. // This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) #pragma once @@ -58,7 +58,7 @@ class FPoseAIUdpSocketReceiver { check(Socket != nullptr); check(Socket->GetSocketType() == SOCKTYPE_Datagram); - + Reader->SetNumUninitialized(MaxReadBufferSize); SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); } @@ -138,6 +138,7 @@ class FPoseAIUdpSocketReceiver /** Update this socket receiver. */ void Update(const FTimespan& SocketWaitTime) { + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) { return; @@ -151,26 +152,30 @@ class FPoseAIUdpSocketReceiver TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); uint32 Size; - - while (Socket!=nullptr && Socket.IsValid() && Socket->HasPendingData(Size)) + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) { - Reader->SetNumUninitialized(FMath::Min(Size, MaxReadBufferSize)); - - int32 Read = 0; - if (Socket->RecvFrom(Reader->GetData(), Reader->Num(), Read, *Sender)) + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) { - ensure((uint32)Read < MaxReadBufferSize); - Reader->RemoveAt(Read, Reader->Num() - Read, false); - // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs - FString recvMessage; - char* bytedata = (char*)Reader->GetData(); - bytedata[Reader->Num()] = '\0'; - recvMessage = FString(UTF8_TO_TCHAR(bytedata)); + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); } + } - + } protected: diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h new file mode 100644 index 0000000..dfa1571 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -0,0 +1,78 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "CoreGlobals.h" +#include "Widgets/SCompoundWidget.h" +#include "Widgets/SBoxPanel.h" +#include "Widgets/Input/SEditableTextBox.h" +#include "Widgets/Text/STextBlock.h" +#include "Widgets/Input/SButton.h" +#include "Widgets/Input/SCheckBox.h" +#include "Widgets/Layout/SBox.h" +#include "Types/WidgetActiveTimerDelegate.h" +#include "SlateOptMacros.h" +#include "Misc/ConfigCacheIni.h" +#include "LiveLinkLog.h" +#include "iLiveLinkSource.h" +#include "LiveLinkSourceFactory.h" +#include "IPAddress.h" +#include "PoseAIStructs.h" + + +/** + * + */ +class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public FWidgetActiveTimerDelegate +{ +public: + SLATE_BEGIN_ARGS(SPoseAILiveLinkWidget) {} + SLATE_END_ARGS() + + /** Constructs this widget with InArgs */ + void Construct(const FArguments& InArgs); + + void setCallback(ULiveLinkSourceFactory::FOnLiveLinkSourceCreated whenCreated) { callback = whenCreated; } + + static void disableExistingSource(); + static TSharedPtr CreateSource(const FString& port); + +protected: + static TWeakPtr source; + ULiveLinkSourceFactory::FOnLiveLinkSourceCreated callback; + + UPROPERTY(EditAnywhere, Config, Category = Custom) + +private: + const static FString section; + static int32 portNum; + static int32 syncFPS; + static int32 cameraFPS; + static int32 modeIndex; + static int32 rigIndex; + static bool isMirrored; + static bool isIPv6; + + static FPoseAIHandshake GetHandshake(); + void UpdatePort(const FText& InText, ETextCommit::Type type); + void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); + void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); + bool IsPortValid() const; + + FReply OnOkClicked(); + FReply OnToggleModeClicked(); + FReply OnToggleRigClicked(); + + TWeakPtr ipv6CheckBox = nullptr; + TSharedPtr portInput = nullptr; + TSharedPtr syncFpsInput = nullptr; + TSharedPtr cameraFpsInput = nullptr; + TSharedPtr modeInput = nullptr; + TSharedPtr rigInput = nullptr; + TWeakPtr mirroredCheckBox = nullptr; + TWeakPtr mixamoCheckBox = nullptr; + TWeakPtr rootMotionCheckBox = nullptr; + + void ReadCheckBox(TWeakPtr& checkBox, bool& readTo); +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs new file mode 100644 index 0000000..e39bf7b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -0,0 +1,60 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLinkEd : ModuleRules +{ + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.1/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/PoseAILiveLink.uplugin new file mode 100644 index 0000000..3ee635d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/PoseAILiveLink.uplugin @@ -0,0 +1,39 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "3.1.29", + "FriendlyName": "PoseAI LiveLink", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", + "Category": "Animation", + "CreatedBy": "Pose AI Ltd", + "CreatedByURL": "www.pose-ai.com", + "DocsURL": "www.pose-ai.com/unreal-engine-livelink", + "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/3db8f23ad003492eb85a50de31e5bc57", + "SupportURL": "support@pose-ai.com", + "EngineVersion": "5.2.0", + "CanContainContent": false, + "IsBetaVersion": false, + "Installed": true, + "Modules": [ + { + "Name": "PoseAILiveLink", + "Type": "Runtime", + "LoadingPhase": "Default", + "WhitelistPlatforms": [ + "Win64", + "Mac" + ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "LiveLink", + "Enabled": true + } + ] +} \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Resources/Icon128.png new file mode 100644 index 0000000..a5f1494 Binary files /dev/null and b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Resources/Icon128.png differ diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..6042e8f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,107 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp new file mode 100644 index 0000000..6a2d82f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -0,0 +1,43 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEndpoint.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port) { + + FName socketType = NAME_DGram; + FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + + socket->SetReuseAddr(); + int actualSize; + socket->SetReceiveBufferSize(64 * 1024, actualSize); + socket->SetSendBufferSize(64 * 1024, actualSize); + + TSharedRef sender = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr(protocolType); + sender->SetIp(0); + sender->SetPort(port); + socket->Bind(*sender); + return MakeShareable(socket); +} + + +FString FPoseAIEndpoint::ToString() const +{ + return Address->ToString(true); +} + + +void FPoseAIEndpoint::Initialize() +{ + CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp new file mode 100644 index 0000000..440a2df --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -0,0 +1,468 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +void UStepCounter::Halt(bool fade) { + num_ = 0; + tail_ = -1; + if (fade) + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout))); + else { + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout + fadeDurationOnTimeout))); + steps_per_second_ = 0.0f; + distance_per_second_ = 0.0f; + } +} + +float UStepCounter::CheckIfActiveAndFade() { + float time_since_last_step = TimeSinceLastStep(); + if (time_since_last_step > timeout) { + num_ = 0; + tail_ = -1; + return (fadeDurationOnTimeout > 0.0f) ? + FMath::Max(0.0f, 1.0f - (time_since_last_step - timeout) / fadeDurationOnTimeout) + : 0.0f; + } + else + return 1.0f; +} + +float UStepCounter::StepsPerSecond() { + return steps_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::DistancePerSecond() { + return distance_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::LastDistance() { + return (num_ > 0) ? heights_[tail_] : 0.0f; +} + +float UStepCounter::TimeSinceLastStep() { + return (FDateTime::Now() - last_time_).GetTotalSeconds();; +} + +void UStepCounter::RegisterStep(float stepDistance) { + totalSteps++; + totalDistance += stepDistance; + tail_ = (tail_ + 1) % num_to_track; + last_time_ = FDateTime::Now(); + times_[tail_] = last_time_; + heights_[tail_] = stepDistance; + num_ = FGenericPlatformMath::Min(num_ + 1, num_to_track); + + float elapsed_time; + if (num_ < 2) + elapsed_time = 1.0f; + else { + int head_ = (tail_ + 1) % num_; + elapsed_time = (times_[tail_] - times_[head_]).GetTotalSeconds(); + if (elapsed_time <= 0.0) elapsed_time = 1.0f; + } + steps_per_second_ = static_cast(times_.Num()) / elapsed_time; + distance_per_second_ = 0.0f; + for (int h = 0; h < num_; ++h) + distance_per_second_ += heights_[h]; + distance_per_second_ /= elapsed_time; + +} +UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { + timeout = timeoutIn; + fadeDurationOnTimeout = fadeDuration; + return this; +} + + +bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, myIP, portNum, isIPv6); +} + +bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { + InitializeObjects(); + FLiveLinkSubjectName addedSubjectName; + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); +} + +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} + +void UPoseAIMovementComponent::InitializeObjects() { + footsteps = NewObject(); + leftsteps = NewObject()->SetProperties(0.3f, 0.1f); + rightsteps = NewObject()->SetProperties(0.3f, 0.1f); + feetsplits = NewObject(); + armpumps = NewObject(); + armflexes = NewObject(); + armjacks = NewObject()->SetProperties(1.0f, 0.5f); + armflapL = NewObject()->SetProperties(0.5f, 1.5f); + armflapR = NewObject()->SetProperties(0.5f, 1.5f); + jumps = NewObject(); +} + +bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken) { + InitializeObjects(); + bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); + if (success) + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); + return success; +} + +void UPoseAIMovementComponent::RegisterAsFirstAvailable() { + InitializeObjects(); + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentForFirstAvailableSubject(this); +} + +void UPoseAIMovementComponent::CloseSource() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCloseSource(subjectName); +} + +void UPoseAIMovementComponent::Disconnect() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastDisconnect(subjectName); +} + +void UPoseAIMovementComponent::Deregister() { + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, FLiveLinkSubjectName(NAME_None) ,true); + subjectName = FLiveLinkSubjectName(NAME_None); +} + +void UPoseAIMovementComponent::ChangeModelConfig(FPoseAIModelConfig config) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastConfigUpdate(subjectName, config); +} + +void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { + UPoseAIEventDispatcher::GetDispatcher()->SetHandshake(handshake); +} + +void UPoseAIMovementComponent::ScaleMotion(float RigHeight, FVector Scale) { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->rigHeight = RigHeight; + lockedRig->liveValues.scaleMotion = Scale; + } +} + +void UPoseAIMovementComponent::SetLiveCameraRotation(float pitch, float yaw, float roll){ + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + // order changed due to rotated root bone in UE + lockedRig->liveValues.cameraRotation = FRotator(roll, yaw, pitch); + } +} + + +void UPoseAIMovementComponent::UseCurrentPoseToOrientCamera() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig==nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + float pitch = -lockedRig->liveValues.upperBodyLean.Y; + float yaw = -lockedRig->liveValues.chestYaw; + lockedRig->liveValues.cameraRotation = FRotator(0.0f, yaw, pitch); + } +} +void UPoseAIMovementComponent::UseCurrentPoseAsBaseTranslation() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.rootOffset = lockedRig->liveValues.rootTranslation; + } +} + +void UPoseAIMovementComponent::ZeroMotion() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.scaleMotion = FVector3d::Zero(); + } +} + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + +bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); + LastMovementComponent = component; + UPoseAIMovementComponent* existing_component; + if (HasComponent(name, existing_component)) { + if (!siezeIfTaken) return false; + existing_component->Deregister(); + } + + FName prev = component->GetSubjectName(); + componentsByName.Remove(prev); + if (name.Name != NAME_None) + componentsByName.Emplace(name, component); + component->lastFrameReceived = FDateTime::Now(); + component->subjectName = name; + return true; +} + + +UPoseAIEventDispatcher* UPoseAIEventDispatcher::theInstance = nullptr; + + +void UPoseAIEventDispatcher::RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component) { + FLiveLinkSubjectName firstUnbound = GetFirstUnboundSubject(); + if (firstUnbound.Name != NAME_None) + component->RegisterAs(firstUnbound, true); + else + componentQueue.Enqueue(component); +} + +FLiveLinkSubjectName UPoseAIEventDispatcher::GetFirstUnboundSubject(bool excludeIdleSubjects) { + for (auto& elem : knownConnectionsWithTime) { + if (excludeIdleSubjects && (FDateTime::Now() - elem.Value).GetTotalSeconds() > timeoutInSeconds) continue; + UPoseAIMovementComponent* component; + if (HasComponent(elem.Key, component) && component->GetSubjectName() == elem.Key) continue; + return elem.Key; + } + return NAME_None; +} + + +void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName) { + knownConnectionsWithTime.Emplace(subjectName, FDateTime::Now()); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* existing_component; + bool isReconnection = HasComponent(subjectName, existing_component); + if (isReconnection) { + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); + } else if (!componentQueue.IsEmpty()) { + UPoseAIMovementComponent* component; + componentQueue.Dequeue(component); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } + } + subjectConnected.Broadcast(subjectName, isReconnection); + }); +} + +void UPoseAIEventDispatcher::BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config) { + modelConfigUpdate.Broadcast(subjectName, config); +} + +void UPoseAIEventDispatcher::BroadcastDisconnect(const FLiveLinkSubjectName& subjectName) { + disconnect.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastCloseSource(const FLiveLinkSubjectName& subjectName) { + closeSource.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastResetLivePosition() { + resetLivePositionEvent.Broadcast(); +} +void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { + handshakeUpdate.Broadcast(handshake); +} + +void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); + }); +} + +void UPoseAIEventDispatcher::BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, visibilityFlags]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onVisibilityChange.Broadcast(visibilityFlags); + component->visibilityFlags = visibilityFlags; + } + }); +} + +void UPoseAIEventDispatcher::BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, values]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onLiveValues.Broadcast(values); + component->SetLiveValues(values); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->footsteps->RegisterStep(stepHeight); + component->onFootstep.Broadcast(stepHeight, isLeftStep); + } + }); +} +void UPoseAIEventDispatcher::BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->feetsplits->RegisterStep(width); + component->onFeetsplit.Broadcast(width, isExpanding); + } + }); +} +void UPoseAIEventDispatcher::BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armpumps->RegisterStep(stepHeight); + component->onArmpump.Broadcast(stepHeight); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armflexes->RegisterStep(width); + component->onArmflex.Broadcast(width, isExpanding); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isRising]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armjacks->RegisterStep(0.5f); + component->onArmjack.Broadcast(isRising); + } + }); +} + + +void UPoseAIEventDispatcher::BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepLeftFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepRightFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastJumps(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onJump.Broadcast(); + component->jumps->RegisterStep(1.0f); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isCrouching]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onCrouch.Broadcast(isCrouching); + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureLeft.Broadcast(gesture); + if (gesture == 10) { + component->armflapL->RegisterStep(1.0f); + component->onArmflapL.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureRight.Broadcast(gesture); + if (gesture == 10) { + component->armflapR->RegisterStep(1.0f); + component->onArmflapR.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneL.Broadcast(zone); + }); +} + + +void UPoseAIEventDispatcher::BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneR.Broadcast(zone); + }); +} + +void UPoseAIEventDispatcher::BroadcastStationary(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onStationary.Broadcast(); + }); +} + + +bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component) { + if (!componentsByName.Contains(name)) + return false; + component = componentsByName[name]; + return component != nullptr && IsValid(component); +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..fbc882c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..004c575 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + if (jsonPose != nullptr && jsonPose->HasField("Face")) { + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..31b50b0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,169 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..f3dd836 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp new file mode 100644 index 0000000..7c6b37e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -0,0 +1,82 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkRetargetRotations.h" + +#include "BonePose.h" +#include "BoneContainer.h" +#include "Engine/Blueprint.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "LiveLinkTypes.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "Roles/LiveLinkAnimationTypes.h" + + +UPoseAILiveLinkRetargetRotations::UPoseAILiveLinkRetargetRotations(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +#if WITH_EDITOR + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + if (Blueprint) + { + OnBlueprintCompiledDelegate = Blueprint->OnCompiled().AddUObject(this, &UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled); + } +#endif +} + +void UPoseAILiveLinkRetargetRotations::BeginDestroy() +{ +#if WITH_EDITOR + if (OnBlueprintCompiledDelegate.IsValid()) + { + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + check(Blueprint); + Blueprint->OnCompiled().Remove(OnBlueprintCompiledDelegate); + OnBlueprintCompiledDelegate.Reset(); + } +#endif + + Super::BeginDestroy(); +} + +void UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled(UBlueprint* TargetBlueprint) +{ + +} + + +void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) +{ + auto rootTransform = InFrameData->Transforms[0]; + auto rootTranslation = rootTransform.GetTranslation() * scaleTranslation; + auto rootZ = FVector3d(0.0f, 0.0f, rootTranslation.Z); + auto rootXY = FVector3d(rootTranslation.X, rootTranslation.Y, 0.0f); + for (int32 i = 0; i < InSkeletonData->BoneNames.Num(); i++) + { + FName boneName = InSkeletonData->BoneNames[i]; + auto jointTransform = InFrameData->Transforms[i]; + const int32 meshIndex = OutPose.GetBoneContainer().GetPoseBoneIndexForBoneName(boneName); + if (meshIndex != INDEX_NONE) + { + FCompactPoseBoneIndex boneIndex = OutPose.GetBoneContainer().MakeCompactPoseIndex(FMeshPoseBoneIndex(meshIndex)); + if (i == 0) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootXY); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + + else if (i == 1) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootZ); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + else if (boneIndex != INDEX_NONE) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + } + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp new file mode 100644 index 0000000..ff8da5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -0,0 +1,271 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkServer.h" +#include "Async/Async.h" +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); +const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); +const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); +const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); +const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); + + +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; + + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); + + FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); + FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); + serverSocket = BuildUdpSocket(serverName, protocolType, port); + poseAILiveLinkRunnable = MakeShared(port, listener, this); + udpSocketSender = MakeShared(serverSocket, *senderName); + + FString myIP; + if (protocolType == FNetworkProtocolTypes::IPv6) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on IPv6 link-Local address (begins with fe80:) and Port:%d"), port); + } else if (GetIP(myIP)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on %s Port:%d"), *myIP, port); + } else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server but can't determine your IP address. You may not have a valid network adapter.")); + } +} + +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + + +bool PoseAILiveLinkServer::GetIP(FString& myIP) { + bool canBind = false; + TSharedRef localIp = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->GetLocalHostAddr(*GLog, canBind); + + if (localIp->IsValid()) { + myIP = (localIp->ToString(false)); + return true; + } else { + myIP = LOCTEXT("undeterminedIP", "Can't determine your local host IP address").ToString(); + return false; + } +} + +void PoseAILiveLinkServer::CleanUp() { + if (!cleaningUp) { + cleaningUp = true; + CleanUpReceiver(); + CleanUpSender(); + TSharedPtr socketForClosure = serverSocket; + uint32 portnum = port; + // using delayed closure to try to resolve issue where Epic built plugin creates crashes while project plugin doesn't. + Async(EAsyncExecution::Thread, [socketForClosure, portnum]() { + FPlatformProcess::Sleep(2.0); + socketForClosure->Close(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Closed socket on Port:%d"), portnum); + }); + } +} + +void PoseAILiveLinkServer::CleanUpReceiver() { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); + udpSocketReceiver->Stop(); + } + + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); + poseAILiveLinkRunnable.Reset(); + } +} +void PoseAILiveLinkServer::CleanUpSender() { + if (udpSocketSender && udpSocketSender.IsValid()) { + Disconnect(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); + udpSocketSender->Stop(); + udpSocketSender->Exit(); + } +} + + +bool PoseAILiveLinkServer::HasValidConnection() const { + return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; +} + +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + if (cleaningUp) return; + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); + return; + } + + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); + if (HasValidConnection() && !sameAsCurrent) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { + endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint + SendHandshake(); + } + else { //reject + UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); + //consider sending rejected connection a warning message + } + } + else if (!HasValidConnection() && !sameAsCurrent) { // new connection + InitiateConnection(jsonObject, endpointRecv); + + } + else { + if (PoseAIRig::IsFrameData(jsonObject)) { + lastConnection = FDateTime::Now(); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } + } + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message + SendHandshake(); + } + } +} + +void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + FString version; + if (!(jsonObject->TryGetStringField(fieldVersion, version))) { + static const FName NAME_NoAppVersion = "PoseAILiveLink_NoAppVersion"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_NoAppVersion, failKey, TEXT("PoseAI: Incoming connection does not have a hello handshake")); + return; + } + else if (!CheckAppVersion(version)) { + static const FName NAME_AppVersionFail = "PoseAILiveLink_AppVersionFail"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_AppVersionFail, failKey, TEXT("PoseAI: Please update the mobile app to at least version %s"), *requiredMinVersion); + UE_LOG(LogTemp, Error, TEXT("PoseAILiveLink: Unknown app version. Can not safely connect."), *version); + return; + } + FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); + UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } +} + +bool PoseAILiveLinkServer::SendString(FString& message) const { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { + return false; + } +} + + +void PoseAILiveLinkServer::SendHandshake() const { + FString message_string = handshake.ToString(); + if (SendString(message_string)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); + } else { //unsuccessful + static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; + static const FGuid GUID_HandshakeFail = FGuid(); + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); + FLiveLinkLog::WarningOnce(NAME_HandshakeFail, failKey, TEXT("PoseAI: Unable to send the handshake to %s"), *(endpoint.ToString())); + } +} + +void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { + handshake = newHandshake; + if (endpoint.IsValid()) + SendHandshake(); +} + + + +void PoseAILiveLinkServer::Disconnect() { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*disconnect); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + if (udpSocketSender->Send(bytedata, endpoint)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent disconnect request to %s"), *(endpoint.ToString())); + } + else { //unsuccesful + UE_LOG(LogTemp, Display, TEXT("PoseAI: Unable to disconnect from %s"), *(endpoint.ToString())); + } + } +} + + +FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const +{ + FString prettyString; + if (!(jsonObject->TryGetStringField(fieldPrettyName, prettyString))) { + prettyString = "Unknown"; + } + return FName(*(prettyString.Append("@").Append(endpointRecv.Address->ToString(false)))); +} + +bool PoseAILiveLinkServer::CheckAppVersion(FString version) const +{ + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: App version %s vs required version %s."), *version, *requiredMinVersion); + TArray appArray; + version.ParseIntoArray(appArray, TEXT("."), false); + if (appArray.Num() < 3) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: App version %s format error"), *version); + } + + TArray requiredArray; + requiredMinVersion.ParseIntoArray(requiredArray, TEXT("."), false); + + for (int32 i = 0; i < 3; i++) { + int32 app = FCString::Atoi(*appArray[i]); + int32 req = FCString::Atoi(*requiredArray[i]); + if (app < req) + return false; + else if (app > req) + return true; + } + return true; +} + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp new file mode 100644 index 0000000..3eddf9f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -0,0 +1,25 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkSourceFactory.h" +#include "SPoseAILiveLinkWidget.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +TSharedPtr UPoseAILiveLinkSourceFactory::BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const +{ + auto rawWidget = SNew(SPoseAILiveLinkWidget); + rawWidget->setCallback(OnLiveLinkSourceCreated); + TSharedPtr myWidget = rawWidget; + return myWidget; +} + +TSharedPtr< ILiveLinkSource > UPoseAILiveLinkSourceFactory::CreateSource(const FString & ConnectionString) const +{ + return SPoseAILiveLinkWidget::CreateSource(ConnectionString); +} + +FText UPoseAILiveLinkSourceFactory::GetSourceDisplayName() const { return LOCTEXT("Pose AI App", "Pose AI App"); } +FText UPoseAILiveLinkSourceFactory::GetSourceTooltip() const { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp new file mode 100644 index 0000000..ef6cf16 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -0,0 +1,877 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +const FString PoseAIRig::fieldBody = FString(TEXT("Body")); +const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); +const FString PoseAIRig::fieldHandLeft = FString(TEXT("LeftHand")); +const FString PoseAIRig::fieldHandRight = FString(TEXT("RightHand")); +const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); +const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); +const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); +const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); +TMap> PoseAIRig::RigMap = {}; + +bool isDifferentAndSet(int32 newValue, int32& storedValue) { + bool isDifferent = newValue != storedValue; + storedValue = newValue; + return isDifferent; +} + + +PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : + name(name), + rigType(FName(handshake.GetRigString())), + includeHands(handshake.IncludesHands()), + isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { + Configure(); +} + +TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { + TSharedPtr rigPtr; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; + } + + rigPtr->Configure(); + RigMap.Add(name, rigPtr); + return rigPtr; +} + +TWeakPtr PoseAIRig::GetRigFromSubjectName(const FLiveLinkSubjectName& name) { + return RigMap.Contains(name)? RigMap[name] : nullptr; +} + + +FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ + FLiveLinkStaticDataStruct staticData; + staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); + FLiveLinkSkeletonStaticData* skelData = staticData.Cast(); + check(skelData); + skelData->SetBoneNames(jointNames); + skelData->SetBoneParents(parentIndices); + return staticData; +} + +void PoseAIRig::AddBone(FName boneName, FName parentName, FVector translation) { + jointNames.Emplace(boneName); + if (parentName == boneName) { + parentIndices.Emplace(-1); + } else { + parentIndices.Emplace(jointNames.IndexOfByKey(parentName)); + } + boneVectors.Add(boneName,translation); + lastBoneAdded = boneName; +} + +void PoseAIRig::AddBoneToLast(FName boneName, FVector translation) { + AddBone(boneName, lastBoneAdded, translation); +} + +bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) +{ + return (jsonObject->HasField(fieldBody)) || (jsonObject->HasField(fieldHandLeft)) || (jsonObject->HasField(fieldHandRight)); +} + +bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + + double timestamp; + jsonObject->TryGetNumberField("Timestamp", timestamp); + // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. + if (liveValues.timestamp - 600.0 < timestamp && timestamp < liveValues.timestamp) { + return false; + } + liveValues.timestamp = timestamp; + + FString rigStringOut; + if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } + return false; + } + + uint32 packetFormat = 0; + jsonObject->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 1) + ProcessCompactSupplementaryData(jsonObject, data); + else + ProcessVerboseSupplementaryData(jsonObject, data); + + TriggerEvents(); + + data.WorldTime = FPlatformTime::Seconds(); + bool has_processed = (packetFormat == 1) ? ProcessCompactRotations(jsonObject, data) : ProcessVerboseRotations(jsonObject, data); + return has_processed; +} + +void PoseAIRig::TriggerEvents() { + /* trigger various events and update the Pose AI Movement Component */ + if (visibilityFlags.HasChanged()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastVisibilityChange(name, visibilityFlags); + } + if (verbose.Events.Jump.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastJumps(name); + } + if (verbose.Events.Footstep.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.Footstep.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFootsteps(name, height, verbose.Events.Footstep.Magnitude > 0.0f); + } + if (verbose.Events.FeetSplit.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.FeetSplit.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFeetsplits(name, width, verbose.Events.FeetSplit.Magnitude < 0.0f); + } + if (verbose.Events.ArmPump.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.ArmPump.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmpumps(name, height); + } + if (verbose.Events.ArmFlex.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.ArmFlex.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmflexes(name, width, verbose.Events.ArmFlex.Magnitude < 0.0f); + } + if (verbose.Events.SidestepL.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepL(name, verbose.Events.SidestepL.Magnitude < 0.0f); + } + if (verbose.Events.SidestepR.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepR(name, verbose.Events.SidestepR.Magnitude < 0.0f); + } + if (verbose.Events.ArmGestureL.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureL.Current == 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, true); + else if (verbose.Events.ArmGestureL.Current == 51) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, false); + else + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureL(name, verbose.Events.ArmGestureL.Current); + } + if (verbose.Events.ArmGestureR.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureR.Current < 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureR(name, verbose.Events.ArmGestureR.Current); + } + if (isDifferentAndSet(liveValues.handZoneLeft, handZoneL)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneL(name, handZoneL); + } + if (isDifferentAndSet(liveValues.handZoneRight, handZoneR)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneR(name, handZoneR); + } + if (isDifferentAndSet(liveValues.stableFeet, stableFeet)) { + if (stableFeet > 1) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastStationary(name); + } + if (liveValues.isCrouching != isCrouching) { + isCrouching = !isCrouching; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCrouches(name, isCrouching); + } + if (visibilityFlags.isTorso) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastLiveValues(name, liveValues); +} + + +void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) { + FString VisA = (objBody->HasTypedField("VisA")) ? objBody->GetStringField("VisA") : ""; + FString ScaA = (objBody->HasTypedField("ScaA")) ? objBody->GetStringField("ScaA") : ""; + FString VecA = (objBody->HasTypedField("VecA")) ? objBody->GetStringField("VecA") : ""; + FString EveA = (objBody->HasTypedField("EveA")) ? objBody->GetStringField("EveA") : ""; + visibilityFlags.ProcessCompact(VisA); + liveValues.ProcessCompactScalarsBody(ScaA); + liveValues.ProcessCompactVectorsBody(VecA); + verbose.Events.ProcessCompactBody(EveA); + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + + } + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + liveValues.ProcessCompactVectorsHandLeft(objHandLeft); + } + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) { + liveValues.ProcessCompactVectorsHandRight(objHandRight); + } +} + +void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { + if (!isDesktop) { + FVector playerMotion = liveValues.cameraRotation.RotateVector(liveValues.rootTranslation - liveValues.rootOffset) * rigHeight * liveValues.scaleMotion; + data.Transforms[0].SetTranslation(playerMotion); + } +} + +bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + FString rotaBody; + FString rotaHandLeft; + FString rotaHandRight; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotaBody = (objBody->HasTypedField("RotA")) ? objBody->GetStringField("RotA") : ""; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotaHandLeft = (objHandLeft->HasTypedField("RotA")) ? objHandLeft->GetStringField("RotA") : ""; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotaHandRight = (objHandRight->HasTypedField("RotA")) ? objHandRight->GetStringField("RotA") : ""; + + + bool hasProcessedRotations; + + if ((rotaBody.Len() < 8 && cachedPose.Num() < 1) ) { + hasProcessedRotations = false; + } + else if (rotaBody.Len() < 8 ) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + AppendCachedRotations(0, 1, componentRotations, data); + + if (rotaBody.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaBody, flatArray); + FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint + } + else + AppendCachedRotations(1, numBodyJoints, componentRotations, data); + + + if (includeHands) { + if (rotaHandLeft.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandLeft, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints, numBodyJoints + numHandJoints, componentRotations, data); + if (rotaHandRight.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandRight, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints + numHandJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints + numHandJoints, numBodyJoints + 2 * numHandJoints, componentRotations, data); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > rotBody = nullptr; + TSharedPtr < FJsonObject > rotHandLeft = nullptr; + TSharedPtr < FJsonObject > rotHandRight = nullptr; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotBody = (objBody->HasTypedField(fieldRotations)) ? objBody->GetObjectField(fieldRotations) : nullptr; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotHandLeft = (objHandLeft->HasTypedField(fieldRotations)) ? objHandLeft->GetObjectField(fieldRotations) : nullptr; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotHandRight = (objHandRight->HasTypedField(fieldRotations)) ? objHandRight->GetObjectField(fieldRotations) : nullptr; + + + bool hasProcessedRotations; + if (rotBody == nullptr && cachedPose.Num() < 1) { + hasProcessedRotations = false; + } + else if (rotBody == nullptr || !visibilityFlags.isTorso) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + + for (int32 i = 0; i < jointNames.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + FQuat rotation; + const TArray < TSharedPtr < FJsonValue > >* outArray; + FString jointString = jointName.ToString(); + if ((rotBody != nullptr && rotBody->TryGetArrayField(jointString, outArray)) || + (rotHandLeft != nullptr && rotHandLeft->TryGetArrayField(jointString, outArray)) || + (rotHandRight != nullptr && rotHandRight->TryGetArrayField(jointString, outArray))) { + rotation = FQuat((*outArray)[0]->AsNumber(), (*outArray)[1]->AsNumber(), (*outArray)[2]->AsNumber(), (*outArray)[3]->AsNumber()); + } + else if (cachedPose.Num() > i) { + rotation = parentQuat * cachedPose[i].GetRotation(); + } + else { + rotation = FQuat::Identity; + } + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations + data.Transforms.Add(transform); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > scaBody = nullptr; + TSharedPtr < FJsonObject > eveBody = nullptr; + TSharedPtr < FJsonObject > vecBody = nullptr; + TSharedPtr < FJsonObject > vecHandLeft = nullptr; + TSharedPtr < FJsonObject > vecHandRight = nullptr; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + + if (objBody != nullptr && objBody.IsValid()) { + verbose.ProcessJsonObject(objBody); + //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; + } + + liveValues.ProcessVerboseBody(verbose); + + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + if (objHandLeft->HasTypedField("Open")) + liveValues.opennessLeftHand = objHandLeft->GetNumberField("Open"); + } + if (objHandRight != nullptr && objHandRight.IsValid()) { + vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandRight(vecHandRight); + if (objHandRight->HasTypedField("Open")) + liveValues.opennessRightHand = objHandRight->GetNumberField("Open"); + } + + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + visibilityFlags.ProcessVerbose(verbose.Scalars); +} + +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + +void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < begin + quatArray.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + const FQuat& rotation = quatArray[i - begin]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < end; i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FQuat& rotation = (cachedPose.Num() > i) ? parentQuat * cachedPose[i].GetRotation() : FQuat::Identity; + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::CachePose(const TArray& transforms) { + cachedPose = TArray(transforms); +} + +void PoseAIRig::Configure() {} + +void PoseAIRigUE4::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-1.448829, 0.531424, 9.00581)); + AddBoneToLast(TEXT("calf_r"), FVector(42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(10.453837, -16.577854, 0.080156)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-1.448829, 0.531424, -9.00581)); + AddBoneToLast(TEXT("calf_l"), FVector(-42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-10.453837, 16.577854, 0.080156)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(10.808878, 0.851415, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(18.875349, -3.801159, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(13.407329, -0.420477, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(16.558783, 0.355318, 0)); + AddBoneToLast(TEXT("head"), FVector(9.283613, -0.364157, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_03"), FVector(11.883688, 2.732088, -3.781983)); + AddBoneToLast(TEXT("upperarm_l"), FVector(15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(30.33993, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_03"), FVector(11.883688, 2.732102, 3.782003)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-30.33993, 0, 0)); + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("index_01_l"), TEXT("hand_l"), FVector(12.068114, -1.763462, -2.109398)); + AddBoneToLast(TEXT("index_02_l"), FVector(4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(3.39379, 0, 0)); + AddBone(TEXT("middle_01_l"), TEXT("hand_l"), FVector(12.244281, -1.293644, 0.571162)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(3.648844, 0, 0)); + AddBone(TEXT("ring_01_l"), TEXT("hand_l"), FVector(11.497885, -1.753527, 2.846912)); + AddBoneToLast(TEXT("ring_02_l"), FVector(4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(3.476652, 0, 0)); + AddBone(TEXT("pinky_01_l"), TEXT("hand_l"), FVector(10.140665, -2.263151, 4.643148)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(2.985631, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(4.762036, -2.374981, -2.53782)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(4.062171, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("index_01_r"), TEXT("hand_r"), FVector(-12.068114, 1.763462, 2.109398)); + AddBoneToLast(TEXT("index_02_r"), FVector(-4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-3.39379, 0, 0)); + AddBone(TEXT("middle_01_r"), TEXT("hand_r"), FVector(-12.244281, 1.293644, -0.571162)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-3.648844, 0, 0)); + AddBone(TEXT("ring_01_r"), TEXT("hand_r"), FVector(-11.497885, 1.753527, -2.846912)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-3.476652, 0, 0)); + AddBone(TEXT("pinky_01_r"), TEXT("hand_r"), FVector(-10.140665, 2.263151, -4.643148)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-2.985631, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-4.762036, 2.374981, 2.53782)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-4.062171, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + + rig = MakeStaticData(); +} + + +void PoseAIRigMixamo::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, -44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(-0.7, -17.8, -5.8)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, -44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0.7, -17.8, -5.8)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(0, -25.7, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("RightForeArm"), FVector(0, -25.7, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(0, -14.0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(0, -14.0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + + + +void PoseAIRigMetaHuman::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-2.3, 0.4, 9.27)); + AddBoneToLast(TEXT("calf_r"), FVector(41.2, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.0, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(7.1, -14.4, -0.4)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-2.3, 0.4, -9.27)); + AddBoneToLast(TEXT("calf_l"), FVector(-41.2, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.0, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-7.1, 14.4, 0.4)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(3.4, 0.0, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(6.3, 0.0, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(6.9, 0.0, 0)); + AddBoneToLast(TEXT("spine_04"), FVector(8.1, 0.0, 0)); + AddBoneToLast(TEXT("spine_05"), FVector(18.3, 0.0, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(11.6, 1.0, 0)); + AddBoneToLast(TEXT("neck_02"), FVector(5.0, 0, 0)); + AddBoneToLast(TEXT("head"), FVector(5.0, 0, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_05"), FVector(5.5, -0.7, -1.2)); + AddBoneToLast(TEXT("upperarm_l"), FVector(17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(27.0, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_05"), FVector(5.5, -0.7, 1.2)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-27.0, 0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_l"), TEXT("lowerarm_l"), FVector(7.0, 0, 0)); + + AddBone(TEXT("index_metacarpal_l"), TEXT("hand_l"), FVector(3.5, 0.4, -2.1)); + AddBoneToLast(TEXT("index_01_l"), FVector(5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_l"), TEXT("hand_l"), FVector(3.3, 0.3, -0.1)); + AddBoneToLast(TEXT("middle_01_l"), FVector(6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_l"), TEXT("hand_l"), FVector(3.2, -0.2, 1.2)); + AddBoneToLast(TEXT("ring_01_l"), FVector(6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_l"), TEXT("hand_l"), FVector(3.1, -0.7, 2.4)); + AddBoneToLast(TEXT("pinky_01_l"), FVector(5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(1.8, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(2.0, -1.0, -2.6)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(2.7, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_r"), TEXT("lowerarm_r"), FVector(-7.0, 0, 0)); + AddBone(TEXT("index_metacarpal_r"), TEXT("hand_r"), FVector(-3.5, -0.4, 2.1)); + AddBoneToLast(TEXT("index_01_r"), FVector(-5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_r"), TEXT("hand_r"), FVector(-3.3, -0.3, 0.1)); + AddBoneToLast(TEXT("middle_01_r"), FVector(-6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_r"), TEXT("hand_r"), FVector(-3.2, 0.2, -1.2)); + AddBoneToLast(TEXT("ring_01_r"), FVector(-6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_r"), TEXT("hand_r"), FVector(-3.1, 0.7, -2.4)); + AddBoneToLast(TEXT("pinky_01_r"), FVector(-5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-1.8, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-2.0, 1.0, 2.6)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigDazUE::Configure() +{ + //daz has extra joints in the legs + rShinJoint = 5; + lShinJoint = 10; + lowerBodyNumOfJoints = 11; + + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hip"), FVector(0.0, -105.0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, -1.8, 0.0)); + + AddBone(TEXT("rThighBend"), TEXT("pelvis"), FVector(-7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("rThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("rShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("rFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("rToe"), FVector(0.0, 0, 14.0)); + + AddBone(TEXT("lThighBend"), TEXT("pelvis"), FVector(7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("lThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("lShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("lFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("lToe"), FVector(0.0, 0.0, 14.0)); + + AddBone(TEXT("abdomenLower"), TEXT("hip"), FVector(0.0, -1.7, -1.5)); + AddBoneToLast(TEXT("abdomenUpper"), FVector(0.0, -8.2, 1.2)); + AddBoneToLast(TEXT("chestLower"), FVector(0.0, -7.9, -0.4)); + AddBoneToLast(TEXT("chestUpper"), FVector(0.0, -13.1, -3.6)); + AddBoneToLast(TEXT("neckLower"), FVector(0.0, -18.3, -1.5)); + AddBoneToLast(TEXT("neckUpper"), FVector(0.0, -3.5, 1.5)); + AddBoneToLast(TEXT("head"), FVector(0.0, -4.9, -0.5)); + + AddBone(TEXT("lCollar"), TEXT("chestUpper"), FVector(3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("lShldrBend"), FVector(11.9, 1.7, 0)); + AddBoneToLast(TEXT("lShldrTwist"), FVector(11.6, 0, 0)); + AddBoneToLast(TEXT("lForearmBend"), FVector(14.4, -0.2, -0.5)); + + AddBone(TEXT("rCollar"), TEXT("chestUpper"), FVector(-3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("rShldrBend"), FVector(-11.9, 1.7, 0)); + AddBoneToLast(TEXT("rShldrTwist"), FVector(-11.6, 0, 0)); + AddBoneToLast(TEXT("rForearmBend"), FVector(-14.4, -0.2, -0.5)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("lForearmTwist"), TEXT("lForearmBend"), FVector(12.1, 0, 0)); + AddBone(TEXT("lHand"), TEXT("lForearmTwist"), FVector(14.2, 0, -0.3)); + + AddBone(TEXT("lCarpal1"), TEXT("lHand"), FVector(0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("lIndex1"), FVector(7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("lIndex2"), FVector(3.9, 0, 0)); + AddBoneToLast(TEXT("lIndex3"), FVector(2.1, 0, 0)); + AddBone(TEXT("lCarpal2"), TEXT("lHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("lMid1"), FVector(7.5, -0.3, 0)); + AddBoneToLast(TEXT("lMid2"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("lMid3"), FVector(2.5, 0, 0)); + AddBone(TEXT("lCarpal3"), TEXT("lHand"), FVector(0.8, -0.4, -0.8)); + AddBoneToLast(TEXT("lRing1"), FVector(6.9, -0.2, 0.0)); + AddBoneToLast(TEXT("lRing2"), FVector(4.0, 0, 0)); + AddBoneToLast(TEXT("lRing3"), FVector(2.2, 0, 0)); + AddBone(TEXT("lCarpal4"), TEXT("lHand"), FVector(0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("lPinky1"), FVector(6.5, 0.2, 0)); + AddBoneToLast(TEXT("lPinky2"), FVector(2.8, 0, 0)); + AddBoneToLast(TEXT("lPinky3"), FVector(1.7, 0, 0)); + AddBone(TEXT("lThumb1"), TEXT("lHand"), FVector(1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("lThumb2"), FVector(4.1, 0, 0)); + AddBoneToLast(TEXT("lThumb3"), FVector(3.0, 0, 0)); + + AddBone(TEXT("rForearmTwist"), TEXT("rForearmBend"), FVector(-12.1, 0, 0)); + AddBone(TEXT("rHand"), TEXT("rForearmTwist"), FVector(-14.2, 0, -0.3)); + + AddBone(TEXT("rCarpal1"), TEXT("rHand"), FVector(-0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("rIndex1"), FVector(-7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("rIndex2"), FVector(-3.9, 0, 0)); + AddBoneToLast(TEXT("rIndex3"), FVector(-2.1, 0, 0)); + AddBone(TEXT("rCarpal2"), TEXT("rHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("rMid1"), FVector(-7.5, -0.3, 0)); + AddBoneToLast(TEXT("rMid2"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("rMid3"), FVector(-2.5, 0, 0)); + AddBone(TEXT("rCarpal3"), TEXT("rHand"), FVector(-0.8, -0.4, 0.8)); + AddBoneToLast(TEXT("rRing1"), FVector(-6.9, -0.2, 0)); + AddBoneToLast(TEXT("rRing2"), FVector(-4.0, 0, 0)); + AddBoneToLast(TEXT("rRing3"), FVector(-2.2, 0, 0)); + AddBone(TEXT("rCarpal4"), TEXT("rHand"), FVector(-0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("rPinky1"), FVector(-6.5, 0.2, 0)); + AddBoneToLast(TEXT("rPinky2"), FVector(-2.8, 0, 0)); + AddBoneToLast(TEXT("rPinky3"), FVector(-1.7, 0, 0)); + AddBone(TEXT("rThumb1"), TEXT("rHand"), FVector(-1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("rThumb2"), FVector(-4.1, 0, 0)); + AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp new file mode 100644 index 0000000..eec8f4a --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -0,0 +1,387 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIStructs.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +// Utility conversion functions for compact representation and from arrays to vectors + +float UintB64ToUint(char a, char b) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + + return reverse_map[static_cast(a)] * 64 + reverse_map[static_cast(b)]; +} +uint32 UintB64ToUint(char a, char b, char c) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + return reverse_map[static_cast(a)] * 4096 + reverse_map[static_cast(b)] * 64 + reverse_map[static_cast(c)]; +} + +float FixedB64pairToFloat(char a, char b) { + static const float firstByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9384465070835368, 0.9697117733268197, 0.9384465070835368, 0.9384465070835368, 0.9697117733268197, 0.6257938446507083, 0.6570591108939912, 0.688324377137274, 0.7195896433805569, 0.7508549096238397, 0.7821201758671226, 0.8133854421104054, 0.8446507083536883, 0.8759159745969711, 0.907181240840254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, -0.9687347337567171, -0.9374694675134343, -0.9062042012701514, -0.8749389350268686, -0.8436736687835857, -0.8124084025403029, -0.78114313629702, -0.7498778700537372, -0.7186126038104543, -0.6873473375671715, -0.6560820713238886, -0.6248168050806058, -0.5935515388373229, -0.5622862725940401, -0.5310210063507572, -0.49975574010747437, -0.4684904738641915, -0.43722520762090866, -0.4059599413776258, -0.37469467513434296, -0.3434294088910601, -0.31216414264777725, -0.2808988764044944, -0.24963361016121155, -0.2183683439179287, 0.0, 0.0, 0.0, 0.0, 0.9697117733268197, 0.0, -0.18710307767464585, -0.155837811431363, -0.12457254518808014, -0.09330727894479729, -0.06204201270151444, -0.030776746458231585, 0.0004885197850512668, 0.03175378602833412, 0.06301905227161697, 0.09428431851489982, 0.12554958475818268, 0.15681485100146553, 0.18808011724474838, 0.21934538348803123, 0.2506106497313141, 0.28187591597459694, 0.3131411822178798, 0.34440644846116264, 0.3756717147044455, 0.40693698094772834, 0.4382022471910112, 0.46946751343429405, 0.5007327796775769, 0.5319980459208598, 0.5632633121641426, 0.5945285784074255 }; + static const float secondByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.030288226673180263, 0.030776746458231558, 0.030288226673180263, 0.030288226673180263, 0.030776746458231558, 0.025403028822667317, 0.025891548607718612, 0.026380068392769906, 0.0268685881778212, 0.027357107962872496, 0.02784562774792379, 0.028334147532975085, 0.02882266731802638, 0.029311187103077674, 0.02979970688812897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004885197850512946, 0.0009770395701025891, 0.0014655593551538837, 0.0019540791402051783, 0.002442598925256473, 0.0029311187103077674, 0.003419638495359062, 0.0039081582804103565, 0.004396678065461651, 0.004885197850512946, 0.00537371763556424, 0.005862237420615535, 0.006350757205666829, 0.006839276990718124, 0.0073277967757694185, 0.007816316560820713, 0.008304836345872008, 0.008793356130923302, 0.009281875915974597, 0.009770395701025891, 0.010258915486077186, 0.01074743527112848, 0.011235955056179775, 0.01172447484123107, 0.012212994626282364, 0.0, 0.0, 0.0, 0.0, 0.030776746458231558, 0.0, 0.012701514411333659, 0.013190034196384953, 0.013678553981436248, 0.014167073766487542, 0.014655593551538837, 0.015144113336590131, 0.015632633121641426, 0.01612115290669272, 0.016609672691744015, 0.01709819247679531, 0.017586712261846604, 0.0180752320468979, 0.018563751831949193, 0.019052271617000488, 0.019540791402051783, 0.020029311187103077, 0.02051783097215437, 0.021006350757205666, 0.02149487054225696, 0.021983390327308255, 0.02247191011235955, 0.022960429897410845, 0.02344894968246214, 0.023937469467513434, 0.024425989252564728, 0.024914509037616023 }; + return firstByte[static_cast(a)] + secondByte[static_cast(b)]; +} + +void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { + flatArray.Reserve(flatArray.Num() + data.Len() / 2); + for (int i = 0; i + 1 < data.Len(); i += 2) + flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); +} + +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { + quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); + for (int i = 0; i + 3 < flatArray.Num(); i += 4) + quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); +} + + +void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ + fieldVector2D.X = (*value)[0]->AsNumber(); + fieldVector2D.Y = (*value)[1]->AsNumber(); + } +} + +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + +void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { + if (value.Num() >= 2) { + fieldVector2D.X = value[0]; + fieldVector2D.Y = value[1]; + } +} + +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + +void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { + if (newValue != field) + changeFlag = true; + field = newValue; +} +void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { + SetAndCheckForChange(newValue > 0.5f, field, changeFlag); +} +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} + +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} + +FString FPoseAIHandshake::ToString() const { + return FString::Printf( + TEXT("{\"HANDSHAKE\":{" + "\"name\":\"Unreal LiveLink\"," + "\"rig\":\"%s\", " + "\"mode\":\"%s\", " + "\"face\":\"%s\", " + "\"context\":\"%s\", " + "\"whoami\":\"%s\", " + "\"signature\":\"%s\", " + "\"mirror\":\"%s\", " + "\"syncFPS\": %d, " + "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " + "\"packetFormat\": %d" + "}}"), + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), + *whoami, + *signature, + *(YesNoString(isMirrored)), + syncFPS, + cameraFPS, + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) + ); +} + + +bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const +{ + return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; +} + + + +FString FPoseAIModelConfig::ToString() const { + return FString::Printf( + TEXT("{\"CONFIG\":{" + "\"mirror\":\"%s\", " + "\"stepSensitivity\":%f, " + "\"armSensitivity\":%f, " + "\"crouchSensitivity\": %f, " + "\"jumpSensitivity\":%f" + "}}"), + *(YesNoString(isMirrored)), + stepSensitivity, + armSensitivity, + crouchSensitivity, + jumpSensitivity + ); +} + +bool FPoseAIEventPairBase::CheckTriggerAndUpdate() { + bool hasChanged = Count != InternalCount; + InternalCount = Count; + return hasChanged; +} + +void FPoseAIEventPair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Magnitude = FixedB64pairToFloat(compactString[3], compactString[4]); +} + +void FPoseAIGesturePair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Current = UintB64ToUint(compactString[3], compactString[4]); +} + +void FPoseAIEventStruct::ProcessCompactBody(const FString& compactString) { + TArray compactOrder = { &Footstep, &SidestepL, &SidestepR, &Jump, &FeetSplit, &ArmPump, &ArmFlex, &ArmGestureL, &ArmGestureR}; + if (compactString.Len() % 5 != 0) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: Invalid event string: %s."), *compactString); + return; + } + for (int i = 0; i < compactOrder.Num(); ++i) { + if (compactString.Len() < 5 * i + 5) + break; + compactOrder[i]->ProcessCompact(compactString.Mid(i * 5, 5)); + } +} + +void FPoseAIVisibilityFlags::ProcessCompact(const FString& visString) { + hasChanged = false; + SetAndCheckForChange(visString[0] != '0', isTorso, hasChanged); + SetAndCheckForChange(visString[1] != '0', isLeftLeg, hasChanged); + SetAndCheckForChange(visString[2] != '0', isRightLeg, hasChanged); + SetAndCheckForChange(visString[3] != '0', isLeftArm, hasChanged); + SetAndCheckForChange(visString[4] != '0', isRightArm, hasChanged); + if (visString.Len() > 5) + SetAndCheckForChange(visString[5] != '0', isFace, hasChanged); +} + +void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) { + int32 idx = 0; + if(compactString.Len() < 14) return; + bodyHeight = FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) + 1.0f; + chestYaw = FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f; + stanceYaw = FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 180.0f; + stableFeet = UintB64ToUint(compactString[idx + 6], compactString[idx + 7]); + handZoneLeft = UintB64ToUint(compactString[idx + 8], compactString[idx + 9]); + handZoneRight = UintB64ToUint(compactString[idx + 10], compactString[idx + 11]); + isCrouching = UintB64ToUint(compactString[idx + 12], compactString[idx + 13]) > 0; +} + +void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { + //tbd - this could be simplified if we don't need to keep supported older versions of the api + int32 idx = 0; + if (compactString.Len() < 12) return; + upperBodyLean.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 180.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f + ); + idx += 4; + hipScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx+1]), + FixedB64pairToFloat(compactString[idx+2], compactString[idx+3]) + ); + idx += 4; + chestScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]), + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) + ); + idx += 4; + if (compactString.Len() < idx + 12) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + handIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + if (compactString.Len() < idx + 18) return; + rootTranslation.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; +} + +void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessLeftHand = handObj->GetNumberField("Open"); +} + +void FPoseAILiveValues::ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessRightHand = handObj->GetNumberField("Open"); +} + + +void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { + hasChanged = false; + SetAndCheckForChange(scalars.VisTorso, isTorso, hasChanged); + SetAndCheckForChange(scalars.VisLegL, isLeftLeg, hasChanged); + SetAndCheckForChange(scalars.VisLegR, isRightLeg, hasChanged); + SetAndCheckForChange(scalars.VisArmL, isLeftArm, hasChanged); + SetAndCheckForChange(scalars.VisArmR, isRightArm, hasChanged); +} + + +const FString FPoseAILiveValues::fieldPointScreen = FString(TEXT("PointScreen")); +const FString FPoseAILiveValues::fieldThumbScreen = FString(TEXT("ThumbScreen")); + +void FPoseAIScalarStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody) { + FJsonObjectConverter::JsonObjectToUStruct(scaBody.ToSharedRef(), this); +} + +void FPoseAIEventStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody) { + FJsonObjectConverter::JsonObjectToUStruct(eveBody.ToSharedRef(), this); +} + +void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) { + FJsonObjectConverter::JsonObjectToUStruct(jsonObj.ToSharedRef(), this); +} + + +void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ + bodyHeight = verbose.Scalars.BodyHeight; + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); + stanceYaw = verbose.Scalars.StanceYaw * 180.0f; + chestYaw = verbose.Scalars.ChestYaw * 180.0f; + isCrouching = verbose.Scalars.IsCrouching > 0.5f; + handZoneLeft = verbose.Scalars.HandZoneL; + handZoneRight= verbose.Scalars.HandZoneR; + ProcessArrayAsVector2D(verbose.Vectors.HipLean, upperBodyLean); + upperBodyLean *= 180.0f; + ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); + ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); + ProcessArrayAsVector3D(verbose.Vectors.Hip, rootTranslation); + ProcessArrayAsVector3D(verbose.Vectors.FootIkL, footIkL); + ProcessArrayAsVector3D(verbose.Vectors.FootIkR, footIkR); +} + + + +void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); +} + +void FPoseAILiveValues::ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); +} + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp new file mode 100644 index 0000000..70554f9 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -0,0 +1,291 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "SPoseAILiveLinkWidget.h" +#include "PoseAILiveLinkSourceFactory.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +TWeakPtr SPoseAILiveLinkWidget::source = nullptr; + +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking +static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; + +const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; +int32 SPoseAILiveLinkWidget::syncFPS = 60; +int32 SPoseAILiveLinkWidget::cameraFPS = 60; +int32 SPoseAILiveLinkWidget::modeIndex = 0; +int32 SPoseAILiveLinkWidget::rigIndex = 0; +bool SPoseAILiveLinkWidget::isMirrored = false; +bool SPoseAILiveLinkWidget::isIPv6 = false; + +BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION +void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) +{ + GConfig->GetBool(*section, TEXT("isIPv6"), isIPv6, GEditorIni); + GConfig->GetBool(*section, TEXT("isMirrored"), isMirrored, GEditorIni); + + GConfig->GetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + if (modeIndex < 0 || modeIndex >= PoseAI_Modes.Num()) + modeIndex = 0; + GConfig->GetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + if (rigIndex < 0 || rigIndex >= PoseAI_Rigs.Num()) + rigIndex = 0; + + GConfig->GetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + + ChildSlot + [ + SNew(SBox).WidthOverride(300) + [ + SNew(SVerticalBox) + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("UseIPv6", "Connect via an IPv6 socket")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(ipv6CheckBox, SCheckBox) + .IsChecked(isIPv6 ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.75f) + [ + SNew(STextBlock).Text(LOCTEXT("PortIPv4", "Port for IPv4 (0 if unused)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).FillWidth(0.25f) + [ + SAssignNew(portInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdatePort) + .Text(FText::FromString(FString::FromInt(portNum))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.65f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleMode", "Toggle Camera Mode")) + + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.45f) + + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleModeClicked) + [ + SAssignNew(modeInput, STextBlock) + .Text(FText::FromString(PoseAI_Modes[modeIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.6f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleRig", "Toggle rig format")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.4f) + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleRigClicked) + [ + SAssignNew(rigInput, STextBlock) + .Text(FText::FromString(PoseAI_Rigs[rigIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("IsMirrored", "Mirror camera (flip left/right)")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(mirroredCheckBox, SCheckBox) + .IsChecked(isMirrored ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("SyncFPS", "Smoothed FPS (by app)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(syncFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateSyncFPS) + .Text(FText::FromString(FString::FromInt(syncFPS))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("CameraFPS", "Request Camera FPS")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(cameraFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateCameraFPS) + .Text(FText::FromString(FString::FromInt(cameraFPS))) + ] + ] + + SVerticalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).AutoHeight() + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnOkClicked) + [ + SNew(STextBlock) + .Text(LOCTEXT("OK", "OK")) + ] + ] + ] + ]; +} +END_SLATE_FUNCTION_BUILD_OPTIMIZATION + + +void SPoseAILiveLinkWidget::UpdatePort(const FText& InText, ETextCommit::Type type) +{ + portNum = FCString::Atoi(*(InText.ToString())); +} + + +bool SPoseAILiveLinkWidget::IsPortValid() const +{ + if (portNum < 1028 || portNum > 49151) { + FLiveLinkLog::Warning(TEXT("PoseAI: %d is an invalid port number. Set a valid port number (>1028 and <49151)."), portNum); + return false; + } + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); + return false; + } + return true; +} + + +void SPoseAILiveLinkWidget::UpdateSyncFPS(const FText& InText, ETextCommit::Type type) +{ + syncFPS = FCString::Atoi(*(InText.ToString())); + if (syncFPS < 0) { + syncFPS = 0; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + +void SPoseAILiveLinkWidget::UpdateCameraFPS(const FText& InText, ETextCommit::Type type) +{ + cameraFPS = FCString::Atoi(*(InText.ToString())); + if (cameraFPS < 24) { + cameraFPS = 24; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + GConfig->SetInt(*section, TEXT("cameraFPS"), cameraFPS, GEditorIni); + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + + +void SPoseAILiveLinkWidget::disableExistingSource() +{ + TSharedPtr linkSource = source.Pin(); + if (linkSource.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); + } +} +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() +{ + FPoseAIHandshake handshake = FPoseAIHandshake(); + handshake.isMirrored = isMirrored; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); + handshake.syncFPS = syncFPS; + handshake.cameraFPS = cameraFPS; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); +} + +FReply SPoseAILiveLinkWidget::OnToggleModeClicked() +{ + modeIndex = (modeIndex+1) % PoseAI_Modes.Num(); + modeInput->SetText(FText::FromString(PoseAI_Modes[modeIndex])); + GConfig->SetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnToggleRigClicked() +{ + rigIndex = (rigIndex + 1) % PoseAI_Rigs.Num(); + rigInput->SetText(FText::FromString(PoseAI_Rigs[rigIndex])); + GConfig->SetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnOkClicked() +{ + ReadCheckBox(mirroredCheckBox, isMirrored); + ReadCheckBox(ipv6CheckBox, isIPv6); + + GConfig->SetBool(*section, TEXT("isMirror"), isMirrored, GEditorIni); + + if (IsPortValid()) { + GConfig->SetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + GConfig->Flush(false, GEditorIni); + FString connectionString = ""; + TSharedPtr src = CreateSource(connectionString); + callback.Execute(src, connectionString); + FLiveLinkLog::Info( + TEXT("PoseAI: Setup source. Rig is in %s format, %s and %s."), + *PoseAI_Rigs[rigIndex], + *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")) + ); + } + return FReply::Handled(); +} + +void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& readTo) +{ + TSharedPtr pin = checkBox.Pin(); + if (pin) + readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h new file mode 100644 index 0000000..977dea1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -0,0 +1,128 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "IPAddress.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "SocketSubsystem.h" + + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port); + + +/** + * Implements a more generic endpoint to allow for both IPv6 and IPv4 networks, based on Epic Games IPv4 endpoint from the Networking module. + * + * Mainly is a wrapper around an FInternetAddr with the helper functions needed to use the networking code with only minor modifications. + * + */ +struct FPoseAIEndpoint +{ + /** Holds the endpoint's IP address. */ + TSharedPtr Address; + + /** Holds the endpoint's port number. */ + uint16 Port; + +public: + + /** Default constructor. */ + FPoseAIEndpoint() { } + + + /** + * Creates and initializes a new endpoint from a given FInternetAddr object. + * + * Note: this constructor will be removed after the socket subsystem has been refactored. + * + * @param InternetAddr The Internet address. + */ + + + FPoseAIEndpoint(const TSharedPtr& InternetAddr) + { + check(InternetAddr.IsValid()); + + int32 OutPort; + Address = InternetAddr; + InternetAddr->GetPort(OutPort); + Port = OutPort; + } + + bool IsValid() const { return Address != nullptr && Address.IsValid(); } + +public: + + /** + * Compares this endpoint with the given endpoint for equality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are equal, false otherwise. + */ + bool operator==(const FPoseAIEndpoint& Other) const + { + return ((Address == Other.Address)); + } + + /** + * Compares this address with the given endpoint for inequality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are not equal, false otherwise. + */ + bool operator!=(const FPoseAIEndpoint& Other) const + { + return (Address != Other.Address); + } + + +public: + + + /** + * Gets a string representation for this endpoint. + * + * @return String representation. + * @see Parse, ToText + */ + POSEAILIVELINK_API FString ToString() const; + + /** + * Gets the display text representation. + * + * @return Text representation. + * @see ToString + */ + FText ToText() const + { + return FText::FromString(ToString()); + } + + TSharedRef ToInternetAddr() const + { + if (CachedSocketSubsystem == nullptr) + Initialize(); + + check(CachedSocketSubsystem != nullptr && "Networking module not loaded and initialized"); + TSharedRef InternetAddr = CachedSocketSubsystem->CreateInternetAddr(Address->GetProtocolType()); + { + InternetAddr->SetRawIp(Address->GetRawIp()); + InternetAddr->SetPort(Address->GetPort()); + } + + return InternetAddr; + } + +public: + + /** Initializes the IP endpoint functionality. */ + static void Initialize(); + + + +private: + /** ISocketSubsystem::Get() is not thread-safe, so we cache it here. */ + static ISocketSubsystem* CachedSocketSubsystem; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h new file mode 100644 index 0000000..3eecfa0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -0,0 +1,395 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "Async/Async.h" +#include "LiveLinkTypes.h" +#include "PoseAIStructs.h" +#include "PoseAIEventDispatcher.generated.h" + + +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); +DECLARE_MULTICAST_DELEGATE_TwoParams(FPoseAIConfigUpdate, const FLiveLinkSubjectName&, FPoseAIModelConfig); + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAISubjectConnected, const FLiveLinkSubjectName&, SubjectName, bool, isReconnection); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIRegisteredAs, const FLiveLinkSubjectName&, SubjectName, FName, ConnectionName); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIFrameReceived); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIVisibilityChange, const FPoseAIVisibilityFlags&, Flags); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAILiveValuesUpdate, const FPoseAILiveValues&, LiveValues); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootstepEvent, float, height, bool, isLeftFoot); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAISidestepEvent, bool, isLeftStep); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootsplitEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmpumpEvent, float, height); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIArmflexEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmjackEvent,bool, isRising); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIArmflapEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIJumpEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAICrouchEvent, bool, isCrouching); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIHandToZoneEvent, int32, zone); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmGestureEvent, int32, armGesture); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIStationaryEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIResetLivePositionEvent); + + +UCLASS(ClassGroup = (PoseAI)) +class POSEAILIVELINK_API UStepCounter : public UObject +{ + GENERATED_BODY() + +public: + /** Convenience setter for timeout and fadeduration at the same time*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + UStepCounter* SetProperties(float timeoutIn = 0.5f, float fadeDuration = 0.2f); + + /** clears all steps. Optionally fades speed over */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + void Halt(bool fade); + + /** Average step distance in [body width, body height] units per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float DistancePerSecond(); + + /** Average number of steps per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float StepsPerSecond(); + + /** Time since last step in seconds*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float TimeSinceLastStep(); + + /** most recent step distance*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float LastDistance(); + + + /** time since last step when motion begins to slow*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float timeout = 0.5f; + + /** after a next step timeout, the speed is faded to zero over this duration*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float fadeDurationOnTimeout = 0.2f; + + /** total steps registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + int32 totalSteps = 0; + + /** total distance registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + float totalDistance = 0.0f; + + void RegisterStep(float stepDistance); + + UStepCounter(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) + { + + times_.SetNumUninitialized(num_to_track); + heights_.SetNumUninitialized(num_to_track); + } + + +private: + float CheckIfActiveAndFade(); + const int32 num_to_track = 4; + int32 num_ = 0; + int32 tail_ = -1; + + float steps_per_second_ = 0.0f; + float distance_per_second_ = 0.0f; + FDateTime last_time_ = FDateTime::Now(); + TArray times_; + TArray heights_; +}; + + +class UPoseAIEventDispatcher; + +UCLASS(ClassGroup = (PoseAI), meta = (BlueprintSpawnableComponent)) +class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent +{ + GENERATED_BODY() + friend UPoseAIEventDispatcher; + + public: + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple portss (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum=8080, bool isIPv6 = false); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + + /** Sends a message to the connected PoseCamera to reconfigure the model with user settings */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void ChangeModelConfig(FPoseAIModelConfig config); + + /** sends disconnect message to app and closes source, freeing up Port*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(); + + /** sends disconnect message to app but does not clsoe source */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Disconnect(); + + /** Get the LiveLink subject name associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectName() { return subjectName; } + + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void RegisterAsFirstAvailable(); + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + static void SetHandshake(const FPoseAIHandshake& handshake); + + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken = true); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Deregister(); + + /** Sets rig height, which scales root motion, and allows scaling per dimension (i.e. set Y=0 for no motion to/from camera) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ScaleMotion(float RigHeight=170.0f, FVector Scale=FVector(1.0f, 1.0f,1.0f)); + + /** compensates for an active source's camera rotation in the real world */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void SetLiveCameraRotation(float pitch, float yaw = 0.0f, float roll=0.0f); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseAsBaseTranslation(); + + /** Have player stand up straight and face screen as part of configuration */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseToOrientCamera(); + + /** Remove all live root motion (sets scalemotion to zero)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ZeroMotion(); + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FDateTime lastFrameReceived; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAIVisibilityFlags visibilityFlags; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAILiveValues mostRecentValues; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* footsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* leftsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* rightsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* feetsplits; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armpumps ; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflexes; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armjacks; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapL; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapR; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* jumps; + + /********** events ********************/ + + /** when the component succeesfully registered with a connection (i.e. a pose camera joined) */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIRegisteredAs onRegistered; + + /** when any body part visisbility flag changes */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIVisibilityChange onVisibilityChange; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAILiveValuesUpdate onLiveValues; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootstepEvent onFootstep; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootsplitEvent onFeetsplit; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepLeftFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepRightFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmpumpEvent onArmpump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflexEvent onArmflex; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmjackEvent onArmjack; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureLeft; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureRight; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIJumpEvent onJump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAICrouchEvent onCrouch; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIStationaryEvent onStationary; + + void SetLiveValues(FPoseAILiveValues values) { + mostRecentValues = values; + } + + virtual void InitializeComponent() override { + Super::InitializeComponent(); + InitializeObjects(); + } + +private: + FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + + void InitializeObjects(); + +}; + + +/** + * Dispatcher to transmit events to blueprints and c++, can access events from all sources + */ +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAIEventDispatcher : public UObject +{ +GENERATED_BODY() +public: + /** Gets the singleton dispatcher. Use this for binding or to get named components */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + static UPoseAIEventDispatcher* GetDispatcher() { + if (theInstance==nullptr) { + theInstance = NewObject(); + theInstance->AddToRoot(); + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: Creating EventDispatcher.")); + } + return theInstance; + } + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void SetHandshake(const FPoseAIHandshake& handshake); + + FPoseAIHandshakeUpdate handshakeUpdate; + FPoseAIConfigUpdate modelConfigUpdate; + FPoseAIDisconnect disconnect; + FPoseAIDisconnect closeSource; + + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + + /** Convenience event to tell animation blueprin*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + void BroadcastResetLivePosition(); + + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISubjectConnected subjectConnected; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIResetLivePositionEvent resetLivePositionEvent; + + // Connection driven events + void BroadcastCloseSource(const FLiveLinkSubjectName& subjectName); + void BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config); + void BroadcastDisconnect(const FLiveLinkSubjectName& subjectName); + void BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName); + void BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName); + + // Pose Camera driven events + void BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight); + void BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising); + void BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching); + void BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep); + void BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastJumps(const FLiveLinkSubjectName& subjectName); + void BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values); + void BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastStationary(const FLiveLinkSubjectName& subjectName); + void BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags); + + bool RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken); + void RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component); + + bool HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component); + + void BroadcastResetZeroLivePosition(); + +private: + static UPoseAIEventDispatcher* theInstance; + const double timeoutInSeconds = 60.0; + TQueue componentQueue; + UPROPERTY() + TMap componentsByName; + TMap knownConnectionsWithTime; + UPoseAIEventDispatcher() : UObject() {}; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h new file mode 100644 index 0000000..75d849b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..13d5dee --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,105 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..56ac13d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,72 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h new file mode 100644 index 0000000..6fd81a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -0,0 +1,144 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAILiveLinkServer.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + + +struct POSEAILIVELINK_API PoseAIPortRecord { + FGuid source; + FName connectionName; + FLiveLinkSubjectKey subjectKey; +}; + +class PoseAILiveLinkSingleSourceListener; + + +/** + * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. + * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource +{ +public: + + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {} + + // custom methods + static bool GetPortGuid(int32 port, FGuid& fguid); + static bool IsValidPort(int32 port); + static FName GetConnectionName(int32 port); + static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); + static FName SubjectNameFromPort(int32 port); + + void disable(); + FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; + int32 port; + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; + + void AddSubject(); + +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h new file mode 100644 index 0000000..1819982 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -0,0 +1,37 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once +#include "CoreMinimal.h" +#include "LiveLinkRetargetAsset.h" +#include "PoseAILiveLinkRetargetRotations.generated.h" + +// Rretarget asset for data coming from Live Link. Remaps rotations onto all bones and only translations for root and pelvis/hip. +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAILiveLinkRetargetRotations : public ULiveLinkRetargetAsset +{ + GENERATED_UCLASS_BODY() + + virtual ~UPoseAILiveLinkRetargetRotations() {} + + //~ Begin UObject Interface + virtual void BeginDestroy() override; + //~ End UObject Interface + + //~ Begin ULiveLinkRetargetAsset interface + virtual void BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) override; + //~ End ULiveLinkRetargetAsset interface + + // allow user to scale translations for differences in skeleton sizes + UPROPERTY(EditAnywhere, Category = Settings) + float scaleTranslation = 1.0f; + +private: + + void OnBlueprintClassCompiled(UBlueprint* TargetBlueprint); + + +#if WITH_EDITOR + /** Blueprint.OnCompiled delegate handle */ + FDelegateHandle OnBlueprintCompiledDelegate; +#endif +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h new file mode 100644 index 0000000..c6d53c7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -0,0 +1,217 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Runtime/Networking/Public/Networking.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "Runtime/Sockets/Public/SocketSubsystem.h" +#include "HAL/RunnableThread.h" +#include "LiveLinkLog.h" +#include "SocketTypes.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "IPAddress.h" +#include "Json.h" +#include "PoseAIStructs.h" +#include "PoseAIUdpSocketReceiver.h" +#include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" + + +class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; +class FPoseAISocketSender; + +// The networking class needs to be rewritten + +class POSEAILIVELINK_API PoseAILiveLinkServer +{ +public: + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); + + ~PoseAILiveLinkServer() { + CleanUp(); + } + + // utility function that identifies host IPv4 address, to be printed in LiveLink console to help user connect to correct address + static bool GetIP(FString& myIP); + + void CleanUp(); + void Disconnect(); + + TSharedPtr GetSocket() const { return serverSocket; } + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; + void SendHandshake() const; + void SetHandshake(const FPoseAIHandshake& handshake); + // receiver will be set on a runnable thread and set once started + void SetReceiver(TSharedPtr receiver) { udpSocketReceiver = receiver; } + + +private: + const static FString fieldPrettyName; + const static FString fieldVersion; + const static FString fieldUUID; + const static FString fieldRigType; + const static FString requiredMinVersion; + + TSharedPtr listener; + TWeakPtr source_; + FPoseAIHandshake handshake; + FName protocolType; + int32 port; + bool cleaningUp = false; + + // time of last connection. After timeout seconds a newer connection can takeover the port. + FDateTime lastConnection; + const double TIMEOUT_SECONDS = 10.0; + + TSharedPtr serverSocket; + + //used to launch receiver without slowing main thread + TSharedPtr poseAILiveLinkRunnable; + + //Listens for packets + TSharedPtr udpSocketReceiver; + + //sends instructions to paired app + TSharedPtr udpSocketSender; + FPoseAIEndpoint endpoint; + + // disconnect message formatted for Pose AI mobile app + FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); + + void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + + + bool HasValidConnection() const; + + FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; + + // make sure mobile app is sufficiently advanced version as both endpoints of software evolve + bool CheckAppVersion(FString version) const; + + //split clean up routine by component + void CleanUpReceiver(); + void CleanUpSender(); + void CleanUpSocket(); + +}; + + + +class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable +{ +public: + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { + myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); + thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); + } + virtual uint32 Run() override; + +protected: + FString myName; + int32 port; + FRunnableThread* thread = nullptr; +private: + PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; + TSharedPtr udpSocketReceiver; +}; + + + + +// built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update +class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable +{ +public: + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { + thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); + } + + + virtual uint32 Run() override { + while (running && thread == nullptr) { + FPlatformProcess::Sleep(0.2); + } + + while (running ) { + Sleep(true); + while (running && sleeping) { + FPlatformProcess::Sleep(0.005); + } + + } + + thread = nullptr; + return 0; + } + + virtual void Stop() override { + running = false; + if (thread != nullptr) { + Sleep(false); + } + } + + bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) + { + if (running) { + + int32 sent = 0; + if (!Socket) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); + return false; + } + + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); + + if (sent != Data->Num()) + return false; + + Sleep(false); + return true; + } + return false; + } + + void Sleep(bool sleep) { + sleeping = sleep; + if (thread != nullptr) + thread->Suspend(sleep); + } + +protected: + /** The network socket. */ + TSharedPtr Socket; + + /** The thread object. */ + FRunnableThread* thread = nullptr; + + bool running = true; + bool sleeping = false; +}; + + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h new file mode 100644 index 0000000..73f462b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -0,0 +1,30 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "LiveLinkSourceFactory.h" +#include "ILiveLinkSource.h" +#include "PoseAILiveLinkSourceFactory.generated.h" + +/** + * + */ +UCLASS() +class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFactory +{ + GENERATED_BODY() + + UPoseAILiveLinkSourceFactory() { + } + + ~UPoseAILiveLinkSourceFactory () { + } + +public: + virtual TSharedPtr< SWidget > BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const override; + virtual TSharedPtr< ILiveLinkSource > CreateSource(const FString& ConnectionString) const override; + virtual FText GetSourceDisplayName() const override; + virtual FText GetSourceTooltip() const override; + virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h new file mode 100644 index 0000000..1e8e6ac --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -0,0 +1,151 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "ILiveLinkSource.h" +#include "LiveLinkSubjectSettings.h" +#include "Roles/LiveLinkAnimationTypes.h" +#include "Json.h" +#include "PoseAIStructs.h" + +struct POSEAILIVELINK_API Remapping +{ + FName TargetJointName; + FQuat RotAdj; + Remapping(FName TargetJointName, FQuat RotAdj) : TargetJointName(TargetJointName), RotAdj(RotAdj) {}; +}; + + + +/** + * Abstract base class for the different rig formats streamable by Pose AI + */ +class POSEAILIVELINK_API PoseAIRig +{ + public: + FLiveLinkStaticDataStruct MakeStaticData(); + bool ProcessFrame(const TSharedPtr, FLiveLinkAnimationFrameData& data); + static bool IsFrameData(const TSharedPtr jsonObject); + static TSharedPtr PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake); + static TWeakPtr GetRigFromSubjectName(const FLiveLinkSubjectName& name); + FName RigType() { return rigType; } + + FPoseAIVisibilityFlags visibilityFlags; + FPoseAILiveValues liveValues; + FPoseAIScalarStruct scalars; + FPoseAIEventStruct events; + + bool useNextRootAsOffset = false; + //ankle to head top height for scaling PoseAI root motion. + float rigHeight = 170.0f; + + float CameraTilt = 0.0f; + + protected: + FLiveLinkStaticDataStruct rig; + FPoseAIVerbose verbose; + static const FString fieldBody; + static const FString fieldRigType; + static const FString fieldHandLeft; + static const FString fieldHandRight; + static const FString fieldRotations; + static const FString fieldEvents; + static const FString fieldScalars; + static const FString fieldVectors; + + protected: + PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake); + virtual ~PoseAIRig() { + }; + + /* sets up skeletal heirarchy and provides default locations for each transform based on default skeleton (i.e. UE4 Mannequen or male Metahuman). + These bone lengths ensure a sensible animation is created even if user does not retarget from livelink */ + virtual void Configure(); //impure for MacOS compatibility + + FLiveLinkSubjectName name; + FName rigType; + + bool includeHands; + bool isMirrored; + bool isLowerBodyRotated; + bool isDesktop; + int32 numBodyJoints = 21; + int32 numHandJoints = 17; + // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) + int32 lowerBodyNumOfJoints = 8; + + int32 rShinJoint = 3; + int32 lShinJoint = 7; + + bool isCrouching = false; + int32 handZoneL = 5; + int32 handZoneR = 5; + int32 stableFeet = 0; + FVector prevRootTranslation = FVector::ZeroVector; + // store translations of deployed rig + TMap boneVectors; + TArray jointNames; + TArray parentIndices; + TArray cachedPose = {}; + + //temporary variable used for convenience in rig construction + FName lastBoneAdded; + + //extra offset for hip bone to accomodate mesh thickness from bone sockets. + float rootHipOffsetZ = 2.0f; + + void AddBone(FName boneName, FName parentName, FVector translation); + void AddBoneToLast(FName boneName, FVector translation); + void CachePose(const TArray& transforms); + void AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + bool ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + bool ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void TriggerEvents(); + void RotateLowerBody180(TArray& quatArray); + + +private: + static TMap> RigMap; +}; + +class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { + public: + PoseAIRigUE4(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { + public: + PoseAIRigMixamo(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + +class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { +public: + PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { +public: + PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h new file mode 100644 index 0000000..32ae5d2 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -0,0 +1,529 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Json.h" +#include "JsonObjectConverter.h" +#include "PoseAIStructs.generated.h" + + + +/* decoding utilities for compact representation */ +float UintB64ToUint(char a, char b); +uint32 UintB64ToUint(char a, char b, char c); +float FixedB64pairToFloat(char a, char b); +void FStringFixed12ToFloat(const FString& data, TArray& flatArray); +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); + +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + + +/* the handshake configures the main parameters of pose camera*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIHandshake +{ + GENERATED_BODY() + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiAppModes mode = EPoseAiAppModes::Room; + + /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; + + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; + + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; + + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString whoami = ""; + + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString signature = ""; + + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + + + bool operator==(const FPoseAIHandshake& Other) const; + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + + +/*adjusts the sensitivity of PoseAI events*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIModelConfig +{ + GENERATED_BODY() + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float stepSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float armSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float crouchSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float jumpSensitivity = 0.5f; + + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + bool isMirrored = true; + + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + +/** base class for the two event notifications formats sent by camera. All events have a uint count, which upon change signifies a new event has been registered + and a second property, either a float or uint. +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /** number of events registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Count = 0; + + virtual void ProcessCompact(const FString& compactString) {}; + bool CheckTriggerAndUpdate(); +private: + uint32 InternalCount = 0; +}; + +/** +*structure to hold a type of event notification +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**magnitude of event where approrpiate */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + float Magnitude = 0.0f; + + void ProcessCompact(const FString& compactString) override; + +}; + + +USTRUCT() +struct POSEAILIVELINK_API FPoseAIGesturePair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**index code for most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Current = 0; + + void ProcessCompact(const FString& compactString) override; + +}; + + +/** +*structure to receive event notifications +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventStruct +{ + GENERATED_BODY() +public: + /** number of footsteps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Footstep; + + /** number of left foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepL; + + /** number of right foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepR; + + /** number of jumps registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Jump; + + /** number of footsplits registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair FeetSplit; + + /** number of arm pumps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmPump; + + /** number of arm flexes registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmFlex; + + /** number of left or dual arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureL; + + /** number of right arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureR; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody); + void ProcessCompactBody(const FString& compactString); + +}; + +/** +*structure to share additional information +*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIScalarStruct +{ + GENERATED_BODY() +public: + /** visibility flags by body part. Correspond to figure in the app */ + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisTorso = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisFace = 0.0f; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 HandZoneL = 5; + + /** location of right hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PosPeAI") + int32 HandZoneR = 5; + + /** Heading in degrees of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float ChestYaw = 0.0f; + + /** Heading in degrees of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float StanceYaw = 0.0f; + + /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float BodyHeight = 0.0f; + + /** whether subject is crouching */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float IsCrouching = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 StableFoot = 0.0f; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody); + +}; + + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors +{ + GENERATED_BODY() +public: + UPROPERTY() + TArray HipLean; + UPROPERTY() + TArray HipScreen; + UPROPERTY() + TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; + UPROPERTY() + TArray Hip; + UPROPERTY() + TArray FootIkL; + UPROPERTY() + TArray FootIkR; +}; + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerbose +{ + GENERATED_BODY() +public: + UPROPERTY() + FPoseAIEventStruct Events; + UPROPERTY() + FPoseAIScalarStruct Scalars; + UPROPERTY() + FPoseAIVerboseBodyVectors Vectors; + void ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj); + +}; + +/** + * structure to store and expose visibility flags for events alerting programmer if subject is out of camera + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVisibilityFlags +{ + GENERATED_BODY() +public: + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isTorso = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isFace = false; + + bool HasChanged() { return hasChanged; } + void ProcessVerbose(FPoseAIScalarStruct& scalars); + void ProcessCompact(const FString& visString); + +private: + bool hasChanged = false; +}; + +/** + * structure to share additional information + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAILiveValues +{ +GENERATED_BODY() +public: + + /** How much subject is leaning in radians, head-to-hips; x to the side, y forward */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D upperBodyLean = FVector2D(0.0f, 0.0f); + + /** estimated stance height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + bool isCrouching = false; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneLeft = 5; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneRight = 5; + + /** location of subject in camera frame, scaled in body units (i.e. multiply by rig height to translate to game world). Pos Y moves toward camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector rootTranslation = FVector(0.0f, -2.0f, 0.0f); + + /** Heading in radians of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float stanceYaw = 0.0f; + + /** Heading in radians of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float chestYaw = 0.0f; + + /** Heading of flattened left foot to right foot vector relative to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 modelLatency = 0.0f; + + /** timestamp according to pose camera device (CMTime), in seconds */ + double timestamp = 0.0; + + /** DEPR current height of jump in body units */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI_DEPR") + float jumpHeight = 0.0f; + + /** DEPR estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI_DEPR") + float bodyHeight = 0.0f; + + /** location of hips in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D hipScreen = FVector2D(0.0f, 0.0f); + + /** location of chest in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D chestScreen = FVector2D(0.0f, 0.0f); + + /** location of left index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandLeft = FVector2D(0.0f, 0.0f); + + /** location of right index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + + /** location of left thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbLeft = FVector2D(0.0f, 0.0f); + + /** location of right thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbRight = FVector2D(0.0f, 0.0f); + + /** how open the left hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessLeftHand = 0.5f; + + /** how open the right hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessRightHand = 0.5f; + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); + + /** if at least one foot has been stationary for a few frames */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 stableFeet = 0; + + FVector rootOffset = FVector(0.0f, 0.0f, 0.0f); + FRotator cameraRotation = FRotator(0.0f, 0.0f, 0.0f); + FVector scaleMotion = FVector(1.0f, 0.0f, 1.0f); + + void ProcessVerboseBody(const FPoseAIVerbose& scalars); + void ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand); + void ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); + void ProcessCompactScalarsBody(const FString& compactString); + void ProcessCompactVectorsBody(const FString& compactString); + void ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject >); + void ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject >); + +private: + static const FString fieldPointScreen; + static const FString fieldThumbScreen; + +}; + diff --git a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h similarity index 86% rename from UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h rename to UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h index df42fe5..9434a67 100644 --- a/UnrealEngineAPI/PluginV1.3/4.26/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -1,4 +1,4 @@ -// Pose AI Ltd Copyright 2021 +// Copyright Pose AI Ltd 2022. All Rights Reserved. // This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) #pragma once @@ -58,7 +58,7 @@ class FPoseAIUdpSocketReceiver { check(Socket != nullptr); check(Socket->GetSocketType() == SOCKTYPE_Datagram); - + Reader->SetNumUninitialized(MaxReadBufferSize); SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); } @@ -138,6 +138,7 @@ class FPoseAIUdpSocketReceiver /** Update this socket receiver. */ void Update(const FTimespan& SocketWaitTime) { + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) { return; @@ -151,26 +152,30 @@ class FPoseAIUdpSocketReceiver TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); uint32 Size; - - while (Socket!=nullptr && Socket.IsValid() && Socket->HasPendingData(Size)) + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) { - Reader->SetNumUninitialized(FMath::Min(Size, MaxReadBufferSize)); - - int32 Read = 0; - if (Socket->RecvFrom(Reader->GetData(), Reader->Num(), Read, *Sender)) + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) { - ensure((uint32)Read < MaxReadBufferSize); - Reader->RemoveAt(Read, Reader->Num() - Read, false); - // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs - FString recvMessage; - char* bytedata = (char*)Reader->GetData(); - bytedata[Reader->Num()] = '\0'; - recvMessage = FString(UTF8_TO_TCHAR(bytedata)); + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); } + } - + } protected: diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h new file mode 100644 index 0000000..dfa1571 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -0,0 +1,78 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "CoreGlobals.h" +#include "Widgets/SCompoundWidget.h" +#include "Widgets/SBoxPanel.h" +#include "Widgets/Input/SEditableTextBox.h" +#include "Widgets/Text/STextBlock.h" +#include "Widgets/Input/SButton.h" +#include "Widgets/Input/SCheckBox.h" +#include "Widgets/Layout/SBox.h" +#include "Types/WidgetActiveTimerDelegate.h" +#include "SlateOptMacros.h" +#include "Misc/ConfigCacheIni.h" +#include "LiveLinkLog.h" +#include "iLiveLinkSource.h" +#include "LiveLinkSourceFactory.h" +#include "IPAddress.h" +#include "PoseAIStructs.h" + + +/** + * + */ +class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public FWidgetActiveTimerDelegate +{ +public: + SLATE_BEGIN_ARGS(SPoseAILiveLinkWidget) {} + SLATE_END_ARGS() + + /** Constructs this widget with InArgs */ + void Construct(const FArguments& InArgs); + + void setCallback(ULiveLinkSourceFactory::FOnLiveLinkSourceCreated whenCreated) { callback = whenCreated; } + + static void disableExistingSource(); + static TSharedPtr CreateSource(const FString& port); + +protected: + static TWeakPtr source; + ULiveLinkSourceFactory::FOnLiveLinkSourceCreated callback; + + UPROPERTY(EditAnywhere, Config, Category = Custom) + +private: + const static FString section; + static int32 portNum; + static int32 syncFPS; + static int32 cameraFPS; + static int32 modeIndex; + static int32 rigIndex; + static bool isMirrored; + static bool isIPv6; + + static FPoseAIHandshake GetHandshake(); + void UpdatePort(const FText& InText, ETextCommit::Type type); + void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); + void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); + bool IsPortValid() const; + + FReply OnOkClicked(); + FReply OnToggleModeClicked(); + FReply OnToggleRigClicked(); + + TWeakPtr ipv6CheckBox = nullptr; + TSharedPtr portInput = nullptr; + TSharedPtr syncFpsInput = nullptr; + TSharedPtr cameraFpsInput = nullptr; + TSharedPtr modeInput = nullptr; + TSharedPtr rigInput = nullptr; + TWeakPtr mirroredCheckBox = nullptr; + TWeakPtr mixamoCheckBox = nullptr; + TWeakPtr rootMotionCheckBox = nullptr; + + void ReadCheckBox(TWeakPtr& checkBox, bool& readTo); +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs new file mode 100644 index 0000000..e39bf7b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -0,0 +1,60 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLinkEd : ModuleRules +{ + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.2/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/PoseAILiveLink.uplugin new file mode 100644 index 0000000..f01285c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/PoseAILiveLink.uplugin @@ -0,0 +1,39 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "3.1.29", + "FriendlyName": "PoseAI LiveLink", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", + "Category": "Animation", + "CreatedBy": "Pose AI Ltd", + "CreatedByURL": "www.pose-ai.com", + "DocsURL": "www.pose-ai.com/unreal-engine-livelink", + "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/3db8f23ad003492eb85a50de31e5bc57", + "SupportURL": "support@pose-ai.com", + "EngineVersion": "5.3.0", + "CanContainContent": false, + "IsBetaVersion": false, + "Installed": true, + "Modules": [ + { + "Name": "PoseAILiveLink", + "Type": "Runtime", + "LoadingPhase": "Default", + "WhitelistPlatforms": [ + "Win64", + "Mac" + ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "LiveLink", + "Enabled": true + } + ] +} \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Resources/Icon128.png new file mode 100644 index 0000000..a5f1494 Binary files /dev/null and b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Resources/Icon128.png differ diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..6042e8f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,107 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp new file mode 100644 index 0000000..6a2d82f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -0,0 +1,43 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEndpoint.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port) { + + FName socketType = NAME_DGram; + FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + + socket->SetReuseAddr(); + int actualSize; + socket->SetReceiveBufferSize(64 * 1024, actualSize); + socket->SetSendBufferSize(64 * 1024, actualSize); + + TSharedRef sender = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr(protocolType); + sender->SetIp(0); + sender->SetPort(port); + socket->Bind(*sender); + return MakeShareable(socket); +} + + +FString FPoseAIEndpoint::ToString() const +{ + return Address->ToString(true); +} + + +void FPoseAIEndpoint::Initialize() +{ + CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp new file mode 100644 index 0000000..440a2df --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -0,0 +1,468 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +void UStepCounter::Halt(bool fade) { + num_ = 0; + tail_ = -1; + if (fade) + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout))); + else { + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout + fadeDurationOnTimeout))); + steps_per_second_ = 0.0f; + distance_per_second_ = 0.0f; + } +} + +float UStepCounter::CheckIfActiveAndFade() { + float time_since_last_step = TimeSinceLastStep(); + if (time_since_last_step > timeout) { + num_ = 0; + tail_ = -1; + return (fadeDurationOnTimeout > 0.0f) ? + FMath::Max(0.0f, 1.0f - (time_since_last_step - timeout) / fadeDurationOnTimeout) + : 0.0f; + } + else + return 1.0f; +} + +float UStepCounter::StepsPerSecond() { + return steps_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::DistancePerSecond() { + return distance_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::LastDistance() { + return (num_ > 0) ? heights_[tail_] : 0.0f; +} + +float UStepCounter::TimeSinceLastStep() { + return (FDateTime::Now() - last_time_).GetTotalSeconds();; +} + +void UStepCounter::RegisterStep(float stepDistance) { + totalSteps++; + totalDistance += stepDistance; + tail_ = (tail_ + 1) % num_to_track; + last_time_ = FDateTime::Now(); + times_[tail_] = last_time_; + heights_[tail_] = stepDistance; + num_ = FGenericPlatformMath::Min(num_ + 1, num_to_track); + + float elapsed_time; + if (num_ < 2) + elapsed_time = 1.0f; + else { + int head_ = (tail_ + 1) % num_; + elapsed_time = (times_[tail_] - times_[head_]).GetTotalSeconds(); + if (elapsed_time <= 0.0) elapsed_time = 1.0f; + } + steps_per_second_ = static_cast(times_.Num()) / elapsed_time; + distance_per_second_ = 0.0f; + for (int h = 0; h < num_; ++h) + distance_per_second_ += heights_[h]; + distance_per_second_ /= elapsed_time; + +} +UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { + timeout = timeoutIn; + fadeDurationOnTimeout = fadeDuration; + return this; +} + + +bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, myIP, portNum, isIPv6); +} + +bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { + InitializeObjects(); + FLiveLinkSubjectName addedSubjectName; + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); +} + +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} + +void UPoseAIMovementComponent::InitializeObjects() { + footsteps = NewObject(); + leftsteps = NewObject()->SetProperties(0.3f, 0.1f); + rightsteps = NewObject()->SetProperties(0.3f, 0.1f); + feetsplits = NewObject(); + armpumps = NewObject(); + armflexes = NewObject(); + armjacks = NewObject()->SetProperties(1.0f, 0.5f); + armflapL = NewObject()->SetProperties(0.5f, 1.5f); + armflapR = NewObject()->SetProperties(0.5f, 1.5f); + jumps = NewObject(); +} + +bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken) { + InitializeObjects(); + bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); + if (success) + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); + return success; +} + +void UPoseAIMovementComponent::RegisterAsFirstAvailable() { + InitializeObjects(); + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentForFirstAvailableSubject(this); +} + +void UPoseAIMovementComponent::CloseSource() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCloseSource(subjectName); +} + +void UPoseAIMovementComponent::Disconnect() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastDisconnect(subjectName); +} + +void UPoseAIMovementComponent::Deregister() { + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, FLiveLinkSubjectName(NAME_None) ,true); + subjectName = FLiveLinkSubjectName(NAME_None); +} + +void UPoseAIMovementComponent::ChangeModelConfig(FPoseAIModelConfig config) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastConfigUpdate(subjectName, config); +} + +void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { + UPoseAIEventDispatcher::GetDispatcher()->SetHandshake(handshake); +} + +void UPoseAIMovementComponent::ScaleMotion(float RigHeight, FVector Scale) { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->rigHeight = RigHeight; + lockedRig->liveValues.scaleMotion = Scale; + } +} + +void UPoseAIMovementComponent::SetLiveCameraRotation(float pitch, float yaw, float roll){ + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + // order changed due to rotated root bone in UE + lockedRig->liveValues.cameraRotation = FRotator(roll, yaw, pitch); + } +} + + +void UPoseAIMovementComponent::UseCurrentPoseToOrientCamera() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig==nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + float pitch = -lockedRig->liveValues.upperBodyLean.Y; + float yaw = -lockedRig->liveValues.chestYaw; + lockedRig->liveValues.cameraRotation = FRotator(0.0f, yaw, pitch); + } +} +void UPoseAIMovementComponent::UseCurrentPoseAsBaseTranslation() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.rootOffset = lockedRig->liveValues.rootTranslation; + } +} + +void UPoseAIMovementComponent::ZeroMotion() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.scaleMotion = FVector3d::Zero(); + } +} + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + +bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); + LastMovementComponent = component; + UPoseAIMovementComponent* existing_component; + if (HasComponent(name, existing_component)) { + if (!siezeIfTaken) return false; + existing_component->Deregister(); + } + + FName prev = component->GetSubjectName(); + componentsByName.Remove(prev); + if (name.Name != NAME_None) + componentsByName.Emplace(name, component); + component->lastFrameReceived = FDateTime::Now(); + component->subjectName = name; + return true; +} + + +UPoseAIEventDispatcher* UPoseAIEventDispatcher::theInstance = nullptr; + + +void UPoseAIEventDispatcher::RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component) { + FLiveLinkSubjectName firstUnbound = GetFirstUnboundSubject(); + if (firstUnbound.Name != NAME_None) + component->RegisterAs(firstUnbound, true); + else + componentQueue.Enqueue(component); +} + +FLiveLinkSubjectName UPoseAIEventDispatcher::GetFirstUnboundSubject(bool excludeIdleSubjects) { + for (auto& elem : knownConnectionsWithTime) { + if (excludeIdleSubjects && (FDateTime::Now() - elem.Value).GetTotalSeconds() > timeoutInSeconds) continue; + UPoseAIMovementComponent* component; + if (HasComponent(elem.Key, component) && component->GetSubjectName() == elem.Key) continue; + return elem.Key; + } + return NAME_None; +} + + +void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName) { + knownConnectionsWithTime.Emplace(subjectName, FDateTime::Now()); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* existing_component; + bool isReconnection = HasComponent(subjectName, existing_component); + if (isReconnection) { + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); + } else if (!componentQueue.IsEmpty()) { + UPoseAIMovementComponent* component; + componentQueue.Dequeue(component); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } + } + subjectConnected.Broadcast(subjectName, isReconnection); + }); +} + +void UPoseAIEventDispatcher::BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config) { + modelConfigUpdate.Broadcast(subjectName, config); +} + +void UPoseAIEventDispatcher::BroadcastDisconnect(const FLiveLinkSubjectName& subjectName) { + disconnect.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastCloseSource(const FLiveLinkSubjectName& subjectName) { + closeSource.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastResetLivePosition() { + resetLivePositionEvent.Broadcast(); +} +void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { + handshakeUpdate.Broadcast(handshake); +} + +void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); + }); +} + +void UPoseAIEventDispatcher::BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, visibilityFlags]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onVisibilityChange.Broadcast(visibilityFlags); + component->visibilityFlags = visibilityFlags; + } + }); +} + +void UPoseAIEventDispatcher::BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, values]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onLiveValues.Broadcast(values); + component->SetLiveValues(values); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->footsteps->RegisterStep(stepHeight); + component->onFootstep.Broadcast(stepHeight, isLeftStep); + } + }); +} +void UPoseAIEventDispatcher::BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->feetsplits->RegisterStep(width); + component->onFeetsplit.Broadcast(width, isExpanding); + } + }); +} +void UPoseAIEventDispatcher::BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armpumps->RegisterStep(stepHeight); + component->onArmpump.Broadcast(stepHeight); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armflexes->RegisterStep(width); + component->onArmflex.Broadcast(width, isExpanding); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isRising]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armjacks->RegisterStep(0.5f); + component->onArmjack.Broadcast(isRising); + } + }); +} + + +void UPoseAIEventDispatcher::BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepLeftFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepRightFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastJumps(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onJump.Broadcast(); + component->jumps->RegisterStep(1.0f); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isCrouching]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onCrouch.Broadcast(isCrouching); + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureLeft.Broadcast(gesture); + if (gesture == 10) { + component->armflapL->RegisterStep(1.0f); + component->onArmflapL.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureRight.Broadcast(gesture); + if (gesture == 10) { + component->armflapR->RegisterStep(1.0f); + component->onArmflapR.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneL.Broadcast(zone); + }); +} + + +void UPoseAIEventDispatcher::BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneR.Broadcast(zone); + }); +} + +void UPoseAIEventDispatcher::BroadcastStationary(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onStationary.Broadcast(); + }); +} + + +bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component) { + if (!componentsByName.Contains(name)) + return false; + component = componentsByName[name]; + return component != nullptr && IsValid(component); +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..fbc882c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..004c575 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + if (jsonPose != nullptr && jsonPose->HasField("Face")) { + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..31b50b0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,169 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..f3dd836 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp new file mode 100644 index 0000000..7c6b37e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -0,0 +1,82 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkRetargetRotations.h" + +#include "BonePose.h" +#include "BoneContainer.h" +#include "Engine/Blueprint.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "LiveLinkTypes.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "Roles/LiveLinkAnimationTypes.h" + + +UPoseAILiveLinkRetargetRotations::UPoseAILiveLinkRetargetRotations(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +#if WITH_EDITOR + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + if (Blueprint) + { + OnBlueprintCompiledDelegate = Blueprint->OnCompiled().AddUObject(this, &UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled); + } +#endif +} + +void UPoseAILiveLinkRetargetRotations::BeginDestroy() +{ +#if WITH_EDITOR + if (OnBlueprintCompiledDelegate.IsValid()) + { + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + check(Blueprint); + Blueprint->OnCompiled().Remove(OnBlueprintCompiledDelegate); + OnBlueprintCompiledDelegate.Reset(); + } +#endif + + Super::BeginDestroy(); +} + +void UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled(UBlueprint* TargetBlueprint) +{ + +} + + +void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) +{ + auto rootTransform = InFrameData->Transforms[0]; + auto rootTranslation = rootTransform.GetTranslation() * scaleTranslation; + auto rootZ = FVector3d(0.0f, 0.0f, rootTranslation.Z); + auto rootXY = FVector3d(rootTranslation.X, rootTranslation.Y, 0.0f); + for (int32 i = 0; i < InSkeletonData->BoneNames.Num(); i++) + { + FName boneName = InSkeletonData->BoneNames[i]; + auto jointTransform = InFrameData->Transforms[i]; + const int32 meshIndex = OutPose.GetBoneContainer().GetPoseBoneIndexForBoneName(boneName); + if (meshIndex != INDEX_NONE) + { + FCompactPoseBoneIndex boneIndex = OutPose.GetBoneContainer().MakeCompactPoseIndex(FMeshPoseBoneIndex(meshIndex)); + if (i == 0) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootXY); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + + else if (i == 1) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootZ); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + else if (boneIndex != INDEX_NONE) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + } + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp new file mode 100644 index 0000000..ff8da5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -0,0 +1,271 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkServer.h" +#include "Async/Async.h" +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); +const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); +const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); +const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); +const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); + + +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; + + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); + + FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); + FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); + serverSocket = BuildUdpSocket(serverName, protocolType, port); + poseAILiveLinkRunnable = MakeShared(port, listener, this); + udpSocketSender = MakeShared(serverSocket, *senderName); + + FString myIP; + if (protocolType == FNetworkProtocolTypes::IPv6) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on IPv6 link-Local address (begins with fe80:) and Port:%d"), port); + } else if (GetIP(myIP)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on %s Port:%d"), *myIP, port); + } else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server but can't determine your IP address. You may not have a valid network adapter.")); + } +} + +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + + +bool PoseAILiveLinkServer::GetIP(FString& myIP) { + bool canBind = false; + TSharedRef localIp = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->GetLocalHostAddr(*GLog, canBind); + + if (localIp->IsValid()) { + myIP = (localIp->ToString(false)); + return true; + } else { + myIP = LOCTEXT("undeterminedIP", "Can't determine your local host IP address").ToString(); + return false; + } +} + +void PoseAILiveLinkServer::CleanUp() { + if (!cleaningUp) { + cleaningUp = true; + CleanUpReceiver(); + CleanUpSender(); + TSharedPtr socketForClosure = serverSocket; + uint32 portnum = port; + // using delayed closure to try to resolve issue where Epic built plugin creates crashes while project plugin doesn't. + Async(EAsyncExecution::Thread, [socketForClosure, portnum]() { + FPlatformProcess::Sleep(2.0); + socketForClosure->Close(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Closed socket on Port:%d"), portnum); + }); + } +} + +void PoseAILiveLinkServer::CleanUpReceiver() { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); + udpSocketReceiver->Stop(); + } + + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); + poseAILiveLinkRunnable.Reset(); + } +} +void PoseAILiveLinkServer::CleanUpSender() { + if (udpSocketSender && udpSocketSender.IsValid()) { + Disconnect(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); + udpSocketSender->Stop(); + udpSocketSender->Exit(); + } +} + + +bool PoseAILiveLinkServer::HasValidConnection() const { + return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; +} + +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + if (cleaningUp) return; + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); + return; + } + + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); + if (HasValidConnection() && !sameAsCurrent) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { + endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint + SendHandshake(); + } + else { //reject + UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); + //consider sending rejected connection a warning message + } + } + else if (!HasValidConnection() && !sameAsCurrent) { // new connection + InitiateConnection(jsonObject, endpointRecv); + + } + else { + if (PoseAIRig::IsFrameData(jsonObject)) { + lastConnection = FDateTime::Now(); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } + } + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message + SendHandshake(); + } + } +} + +void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + FString version; + if (!(jsonObject->TryGetStringField(fieldVersion, version))) { + static const FName NAME_NoAppVersion = "PoseAILiveLink_NoAppVersion"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_NoAppVersion, failKey, TEXT("PoseAI: Incoming connection does not have a hello handshake")); + return; + } + else if (!CheckAppVersion(version)) { + static const FName NAME_AppVersionFail = "PoseAILiveLink_AppVersionFail"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_AppVersionFail, failKey, TEXT("PoseAI: Please update the mobile app to at least version %s"), *requiredMinVersion); + UE_LOG(LogTemp, Error, TEXT("PoseAILiveLink: Unknown app version. Can not safely connect."), *version); + return; + } + FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); + UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } +} + +bool PoseAILiveLinkServer::SendString(FString& message) const { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { + return false; + } +} + + +void PoseAILiveLinkServer::SendHandshake() const { + FString message_string = handshake.ToString(); + if (SendString(message_string)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); + } else { //unsuccessful + static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; + static const FGuid GUID_HandshakeFail = FGuid(); + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); + FLiveLinkLog::WarningOnce(NAME_HandshakeFail, failKey, TEXT("PoseAI: Unable to send the handshake to %s"), *(endpoint.ToString())); + } +} + +void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { + handshake = newHandshake; + if (endpoint.IsValid()) + SendHandshake(); +} + + + +void PoseAILiveLinkServer::Disconnect() { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*disconnect); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + if (udpSocketSender->Send(bytedata, endpoint)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent disconnect request to %s"), *(endpoint.ToString())); + } + else { //unsuccesful + UE_LOG(LogTemp, Display, TEXT("PoseAI: Unable to disconnect from %s"), *(endpoint.ToString())); + } + } +} + + +FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const +{ + FString prettyString; + if (!(jsonObject->TryGetStringField(fieldPrettyName, prettyString))) { + prettyString = "Unknown"; + } + return FName(*(prettyString.Append("@").Append(endpointRecv.Address->ToString(false)))); +} + +bool PoseAILiveLinkServer::CheckAppVersion(FString version) const +{ + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: App version %s vs required version %s."), *version, *requiredMinVersion); + TArray appArray; + version.ParseIntoArray(appArray, TEXT("."), false); + if (appArray.Num() < 3) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: App version %s format error"), *version); + } + + TArray requiredArray; + requiredMinVersion.ParseIntoArray(requiredArray, TEXT("."), false); + + for (int32 i = 0; i < 3; i++) { + int32 app = FCString::Atoi(*appArray[i]); + int32 req = FCString::Atoi(*requiredArray[i]); + if (app < req) + return false; + else if (app > req) + return true; + } + return true; +} + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp new file mode 100644 index 0000000..3eddf9f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -0,0 +1,25 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkSourceFactory.h" +#include "SPoseAILiveLinkWidget.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +TSharedPtr UPoseAILiveLinkSourceFactory::BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const +{ + auto rawWidget = SNew(SPoseAILiveLinkWidget); + rawWidget->setCallback(OnLiveLinkSourceCreated); + TSharedPtr myWidget = rawWidget; + return myWidget; +} + +TSharedPtr< ILiveLinkSource > UPoseAILiveLinkSourceFactory::CreateSource(const FString & ConnectionString) const +{ + return SPoseAILiveLinkWidget::CreateSource(ConnectionString); +} + +FText UPoseAILiveLinkSourceFactory::GetSourceDisplayName() const { return LOCTEXT("Pose AI App", "Pose AI App"); } +FText UPoseAILiveLinkSourceFactory::GetSourceTooltip() const { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp new file mode 100644 index 0000000..ef6cf16 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -0,0 +1,877 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +const FString PoseAIRig::fieldBody = FString(TEXT("Body")); +const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); +const FString PoseAIRig::fieldHandLeft = FString(TEXT("LeftHand")); +const FString PoseAIRig::fieldHandRight = FString(TEXT("RightHand")); +const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); +const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); +const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); +const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); +TMap> PoseAIRig::RigMap = {}; + +bool isDifferentAndSet(int32 newValue, int32& storedValue) { + bool isDifferent = newValue != storedValue; + storedValue = newValue; + return isDifferent; +} + + +PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : + name(name), + rigType(FName(handshake.GetRigString())), + includeHands(handshake.IncludesHands()), + isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { + Configure(); +} + +TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { + TSharedPtr rigPtr; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; + } + + rigPtr->Configure(); + RigMap.Add(name, rigPtr); + return rigPtr; +} + +TWeakPtr PoseAIRig::GetRigFromSubjectName(const FLiveLinkSubjectName& name) { + return RigMap.Contains(name)? RigMap[name] : nullptr; +} + + +FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ + FLiveLinkStaticDataStruct staticData; + staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); + FLiveLinkSkeletonStaticData* skelData = staticData.Cast(); + check(skelData); + skelData->SetBoneNames(jointNames); + skelData->SetBoneParents(parentIndices); + return staticData; +} + +void PoseAIRig::AddBone(FName boneName, FName parentName, FVector translation) { + jointNames.Emplace(boneName); + if (parentName == boneName) { + parentIndices.Emplace(-1); + } else { + parentIndices.Emplace(jointNames.IndexOfByKey(parentName)); + } + boneVectors.Add(boneName,translation); + lastBoneAdded = boneName; +} + +void PoseAIRig::AddBoneToLast(FName boneName, FVector translation) { + AddBone(boneName, lastBoneAdded, translation); +} + +bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) +{ + return (jsonObject->HasField(fieldBody)) || (jsonObject->HasField(fieldHandLeft)) || (jsonObject->HasField(fieldHandRight)); +} + +bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + + double timestamp; + jsonObject->TryGetNumberField("Timestamp", timestamp); + // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. + if (liveValues.timestamp - 600.0 < timestamp && timestamp < liveValues.timestamp) { + return false; + } + liveValues.timestamp = timestamp; + + FString rigStringOut; + if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } + return false; + } + + uint32 packetFormat = 0; + jsonObject->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 1) + ProcessCompactSupplementaryData(jsonObject, data); + else + ProcessVerboseSupplementaryData(jsonObject, data); + + TriggerEvents(); + + data.WorldTime = FPlatformTime::Seconds(); + bool has_processed = (packetFormat == 1) ? ProcessCompactRotations(jsonObject, data) : ProcessVerboseRotations(jsonObject, data); + return has_processed; +} + +void PoseAIRig::TriggerEvents() { + /* trigger various events and update the Pose AI Movement Component */ + if (visibilityFlags.HasChanged()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastVisibilityChange(name, visibilityFlags); + } + if (verbose.Events.Jump.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastJumps(name); + } + if (verbose.Events.Footstep.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.Footstep.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFootsteps(name, height, verbose.Events.Footstep.Magnitude > 0.0f); + } + if (verbose.Events.FeetSplit.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.FeetSplit.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFeetsplits(name, width, verbose.Events.FeetSplit.Magnitude < 0.0f); + } + if (verbose.Events.ArmPump.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.ArmPump.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmpumps(name, height); + } + if (verbose.Events.ArmFlex.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.ArmFlex.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmflexes(name, width, verbose.Events.ArmFlex.Magnitude < 0.0f); + } + if (verbose.Events.SidestepL.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepL(name, verbose.Events.SidestepL.Magnitude < 0.0f); + } + if (verbose.Events.SidestepR.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepR(name, verbose.Events.SidestepR.Magnitude < 0.0f); + } + if (verbose.Events.ArmGestureL.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureL.Current == 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, true); + else if (verbose.Events.ArmGestureL.Current == 51) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, false); + else + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureL(name, verbose.Events.ArmGestureL.Current); + } + if (verbose.Events.ArmGestureR.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureR.Current < 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureR(name, verbose.Events.ArmGestureR.Current); + } + if (isDifferentAndSet(liveValues.handZoneLeft, handZoneL)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneL(name, handZoneL); + } + if (isDifferentAndSet(liveValues.handZoneRight, handZoneR)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneR(name, handZoneR); + } + if (isDifferentAndSet(liveValues.stableFeet, stableFeet)) { + if (stableFeet > 1) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastStationary(name); + } + if (liveValues.isCrouching != isCrouching) { + isCrouching = !isCrouching; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCrouches(name, isCrouching); + } + if (visibilityFlags.isTorso) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastLiveValues(name, liveValues); +} + + +void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) { + FString VisA = (objBody->HasTypedField("VisA")) ? objBody->GetStringField("VisA") : ""; + FString ScaA = (objBody->HasTypedField("ScaA")) ? objBody->GetStringField("ScaA") : ""; + FString VecA = (objBody->HasTypedField("VecA")) ? objBody->GetStringField("VecA") : ""; + FString EveA = (objBody->HasTypedField("EveA")) ? objBody->GetStringField("EveA") : ""; + visibilityFlags.ProcessCompact(VisA); + liveValues.ProcessCompactScalarsBody(ScaA); + liveValues.ProcessCompactVectorsBody(VecA); + verbose.Events.ProcessCompactBody(EveA); + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + + } + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + liveValues.ProcessCompactVectorsHandLeft(objHandLeft); + } + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) { + liveValues.ProcessCompactVectorsHandRight(objHandRight); + } +} + +void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { + if (!isDesktop) { + FVector playerMotion = liveValues.cameraRotation.RotateVector(liveValues.rootTranslation - liveValues.rootOffset) * rigHeight * liveValues.scaleMotion; + data.Transforms[0].SetTranslation(playerMotion); + } +} + +bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + FString rotaBody; + FString rotaHandLeft; + FString rotaHandRight; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotaBody = (objBody->HasTypedField("RotA")) ? objBody->GetStringField("RotA") : ""; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotaHandLeft = (objHandLeft->HasTypedField("RotA")) ? objHandLeft->GetStringField("RotA") : ""; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotaHandRight = (objHandRight->HasTypedField("RotA")) ? objHandRight->GetStringField("RotA") : ""; + + + bool hasProcessedRotations; + + if ((rotaBody.Len() < 8 && cachedPose.Num() < 1) ) { + hasProcessedRotations = false; + } + else if (rotaBody.Len() < 8 ) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + AppendCachedRotations(0, 1, componentRotations, data); + + if (rotaBody.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaBody, flatArray); + FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint + } + else + AppendCachedRotations(1, numBodyJoints, componentRotations, data); + + + if (includeHands) { + if (rotaHandLeft.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandLeft, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints, numBodyJoints + numHandJoints, componentRotations, data); + if (rotaHandRight.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandRight, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints + numHandJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints + numHandJoints, numBodyJoints + 2 * numHandJoints, componentRotations, data); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > rotBody = nullptr; + TSharedPtr < FJsonObject > rotHandLeft = nullptr; + TSharedPtr < FJsonObject > rotHandRight = nullptr; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotBody = (objBody->HasTypedField(fieldRotations)) ? objBody->GetObjectField(fieldRotations) : nullptr; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotHandLeft = (objHandLeft->HasTypedField(fieldRotations)) ? objHandLeft->GetObjectField(fieldRotations) : nullptr; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotHandRight = (objHandRight->HasTypedField(fieldRotations)) ? objHandRight->GetObjectField(fieldRotations) : nullptr; + + + bool hasProcessedRotations; + if (rotBody == nullptr && cachedPose.Num() < 1) { + hasProcessedRotations = false; + } + else if (rotBody == nullptr || !visibilityFlags.isTorso) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + + for (int32 i = 0; i < jointNames.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + FQuat rotation; + const TArray < TSharedPtr < FJsonValue > >* outArray; + FString jointString = jointName.ToString(); + if ((rotBody != nullptr && rotBody->TryGetArrayField(jointString, outArray)) || + (rotHandLeft != nullptr && rotHandLeft->TryGetArrayField(jointString, outArray)) || + (rotHandRight != nullptr && rotHandRight->TryGetArrayField(jointString, outArray))) { + rotation = FQuat((*outArray)[0]->AsNumber(), (*outArray)[1]->AsNumber(), (*outArray)[2]->AsNumber(), (*outArray)[3]->AsNumber()); + } + else if (cachedPose.Num() > i) { + rotation = parentQuat * cachedPose[i].GetRotation(); + } + else { + rotation = FQuat::Identity; + } + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations + data.Transforms.Add(transform); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > scaBody = nullptr; + TSharedPtr < FJsonObject > eveBody = nullptr; + TSharedPtr < FJsonObject > vecBody = nullptr; + TSharedPtr < FJsonObject > vecHandLeft = nullptr; + TSharedPtr < FJsonObject > vecHandRight = nullptr; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + + if (objBody != nullptr && objBody.IsValid()) { + verbose.ProcessJsonObject(objBody); + //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; + } + + liveValues.ProcessVerboseBody(verbose); + + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + if (objHandLeft->HasTypedField("Open")) + liveValues.opennessLeftHand = objHandLeft->GetNumberField("Open"); + } + if (objHandRight != nullptr && objHandRight.IsValid()) { + vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandRight(vecHandRight); + if (objHandRight->HasTypedField("Open")) + liveValues.opennessRightHand = objHandRight->GetNumberField("Open"); + } + + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + visibilityFlags.ProcessVerbose(verbose.Scalars); +} + +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + +void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < begin + quatArray.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + const FQuat& rotation = quatArray[i - begin]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < end; i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FQuat& rotation = (cachedPose.Num() > i) ? parentQuat * cachedPose[i].GetRotation() : FQuat::Identity; + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::CachePose(const TArray& transforms) { + cachedPose = TArray(transforms); +} + +void PoseAIRig::Configure() {} + +void PoseAIRigUE4::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-1.448829, 0.531424, 9.00581)); + AddBoneToLast(TEXT("calf_r"), FVector(42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(10.453837, -16.577854, 0.080156)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-1.448829, 0.531424, -9.00581)); + AddBoneToLast(TEXT("calf_l"), FVector(-42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-10.453837, 16.577854, 0.080156)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(10.808878, 0.851415, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(18.875349, -3.801159, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(13.407329, -0.420477, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(16.558783, 0.355318, 0)); + AddBoneToLast(TEXT("head"), FVector(9.283613, -0.364157, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_03"), FVector(11.883688, 2.732088, -3.781983)); + AddBoneToLast(TEXT("upperarm_l"), FVector(15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(30.33993, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_03"), FVector(11.883688, 2.732102, 3.782003)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-30.33993, 0, 0)); + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("index_01_l"), TEXT("hand_l"), FVector(12.068114, -1.763462, -2.109398)); + AddBoneToLast(TEXT("index_02_l"), FVector(4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(3.39379, 0, 0)); + AddBone(TEXT("middle_01_l"), TEXT("hand_l"), FVector(12.244281, -1.293644, 0.571162)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(3.648844, 0, 0)); + AddBone(TEXT("ring_01_l"), TEXT("hand_l"), FVector(11.497885, -1.753527, 2.846912)); + AddBoneToLast(TEXT("ring_02_l"), FVector(4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(3.476652, 0, 0)); + AddBone(TEXT("pinky_01_l"), TEXT("hand_l"), FVector(10.140665, -2.263151, 4.643148)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(2.985631, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(4.762036, -2.374981, -2.53782)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(4.062171, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("index_01_r"), TEXT("hand_r"), FVector(-12.068114, 1.763462, 2.109398)); + AddBoneToLast(TEXT("index_02_r"), FVector(-4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-3.39379, 0, 0)); + AddBone(TEXT("middle_01_r"), TEXT("hand_r"), FVector(-12.244281, 1.293644, -0.571162)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-3.648844, 0, 0)); + AddBone(TEXT("ring_01_r"), TEXT("hand_r"), FVector(-11.497885, 1.753527, -2.846912)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-3.476652, 0, 0)); + AddBone(TEXT("pinky_01_r"), TEXT("hand_r"), FVector(-10.140665, 2.263151, -4.643148)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-2.985631, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-4.762036, 2.374981, 2.53782)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-4.062171, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + + rig = MakeStaticData(); +} + + +void PoseAIRigMixamo::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, -44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(-0.7, -17.8, -5.8)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, -44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0.7, -17.8, -5.8)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(0, -25.7, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("RightForeArm"), FVector(0, -25.7, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(0, -14.0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(0, -14.0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + + + +void PoseAIRigMetaHuman::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-2.3, 0.4, 9.27)); + AddBoneToLast(TEXT("calf_r"), FVector(41.2, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.0, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(7.1, -14.4, -0.4)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-2.3, 0.4, -9.27)); + AddBoneToLast(TEXT("calf_l"), FVector(-41.2, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.0, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-7.1, 14.4, 0.4)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(3.4, 0.0, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(6.3, 0.0, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(6.9, 0.0, 0)); + AddBoneToLast(TEXT("spine_04"), FVector(8.1, 0.0, 0)); + AddBoneToLast(TEXT("spine_05"), FVector(18.3, 0.0, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(11.6, 1.0, 0)); + AddBoneToLast(TEXT("neck_02"), FVector(5.0, 0, 0)); + AddBoneToLast(TEXT("head"), FVector(5.0, 0, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_05"), FVector(5.5, -0.7, -1.2)); + AddBoneToLast(TEXT("upperarm_l"), FVector(17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(27.0, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_05"), FVector(5.5, -0.7, 1.2)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-27.0, 0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_l"), TEXT("lowerarm_l"), FVector(7.0, 0, 0)); + + AddBone(TEXT("index_metacarpal_l"), TEXT("hand_l"), FVector(3.5, 0.4, -2.1)); + AddBoneToLast(TEXT("index_01_l"), FVector(5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_l"), TEXT("hand_l"), FVector(3.3, 0.3, -0.1)); + AddBoneToLast(TEXT("middle_01_l"), FVector(6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_l"), TEXT("hand_l"), FVector(3.2, -0.2, 1.2)); + AddBoneToLast(TEXT("ring_01_l"), FVector(6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_l"), TEXT("hand_l"), FVector(3.1, -0.7, 2.4)); + AddBoneToLast(TEXT("pinky_01_l"), FVector(5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(1.8, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(2.0, -1.0, -2.6)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(2.7, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_r"), TEXT("lowerarm_r"), FVector(-7.0, 0, 0)); + AddBone(TEXT("index_metacarpal_r"), TEXT("hand_r"), FVector(-3.5, -0.4, 2.1)); + AddBoneToLast(TEXT("index_01_r"), FVector(-5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_r"), TEXT("hand_r"), FVector(-3.3, -0.3, 0.1)); + AddBoneToLast(TEXT("middle_01_r"), FVector(-6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_r"), TEXT("hand_r"), FVector(-3.2, 0.2, -1.2)); + AddBoneToLast(TEXT("ring_01_r"), FVector(-6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_r"), TEXT("hand_r"), FVector(-3.1, 0.7, -2.4)); + AddBoneToLast(TEXT("pinky_01_r"), FVector(-5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-1.8, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-2.0, 1.0, 2.6)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigDazUE::Configure() +{ + //daz has extra joints in the legs + rShinJoint = 5; + lShinJoint = 10; + lowerBodyNumOfJoints = 11; + + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hip"), FVector(0.0, -105.0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, -1.8, 0.0)); + + AddBone(TEXT("rThighBend"), TEXT("pelvis"), FVector(-7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("rThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("rShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("rFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("rToe"), FVector(0.0, 0, 14.0)); + + AddBone(TEXT("lThighBend"), TEXT("pelvis"), FVector(7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("lThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("lShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("lFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("lToe"), FVector(0.0, 0.0, 14.0)); + + AddBone(TEXT("abdomenLower"), TEXT("hip"), FVector(0.0, -1.7, -1.5)); + AddBoneToLast(TEXT("abdomenUpper"), FVector(0.0, -8.2, 1.2)); + AddBoneToLast(TEXT("chestLower"), FVector(0.0, -7.9, -0.4)); + AddBoneToLast(TEXT("chestUpper"), FVector(0.0, -13.1, -3.6)); + AddBoneToLast(TEXT("neckLower"), FVector(0.0, -18.3, -1.5)); + AddBoneToLast(TEXT("neckUpper"), FVector(0.0, -3.5, 1.5)); + AddBoneToLast(TEXT("head"), FVector(0.0, -4.9, -0.5)); + + AddBone(TEXT("lCollar"), TEXT("chestUpper"), FVector(3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("lShldrBend"), FVector(11.9, 1.7, 0)); + AddBoneToLast(TEXT("lShldrTwist"), FVector(11.6, 0, 0)); + AddBoneToLast(TEXT("lForearmBend"), FVector(14.4, -0.2, -0.5)); + + AddBone(TEXT("rCollar"), TEXT("chestUpper"), FVector(-3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("rShldrBend"), FVector(-11.9, 1.7, 0)); + AddBoneToLast(TEXT("rShldrTwist"), FVector(-11.6, 0, 0)); + AddBoneToLast(TEXT("rForearmBend"), FVector(-14.4, -0.2, -0.5)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("lForearmTwist"), TEXT("lForearmBend"), FVector(12.1, 0, 0)); + AddBone(TEXT("lHand"), TEXT("lForearmTwist"), FVector(14.2, 0, -0.3)); + + AddBone(TEXT("lCarpal1"), TEXT("lHand"), FVector(0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("lIndex1"), FVector(7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("lIndex2"), FVector(3.9, 0, 0)); + AddBoneToLast(TEXT("lIndex3"), FVector(2.1, 0, 0)); + AddBone(TEXT("lCarpal2"), TEXT("lHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("lMid1"), FVector(7.5, -0.3, 0)); + AddBoneToLast(TEXT("lMid2"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("lMid3"), FVector(2.5, 0, 0)); + AddBone(TEXT("lCarpal3"), TEXT("lHand"), FVector(0.8, -0.4, -0.8)); + AddBoneToLast(TEXT("lRing1"), FVector(6.9, -0.2, 0.0)); + AddBoneToLast(TEXT("lRing2"), FVector(4.0, 0, 0)); + AddBoneToLast(TEXT("lRing3"), FVector(2.2, 0, 0)); + AddBone(TEXT("lCarpal4"), TEXT("lHand"), FVector(0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("lPinky1"), FVector(6.5, 0.2, 0)); + AddBoneToLast(TEXT("lPinky2"), FVector(2.8, 0, 0)); + AddBoneToLast(TEXT("lPinky3"), FVector(1.7, 0, 0)); + AddBone(TEXT("lThumb1"), TEXT("lHand"), FVector(1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("lThumb2"), FVector(4.1, 0, 0)); + AddBoneToLast(TEXT("lThumb3"), FVector(3.0, 0, 0)); + + AddBone(TEXT("rForearmTwist"), TEXT("rForearmBend"), FVector(-12.1, 0, 0)); + AddBone(TEXT("rHand"), TEXT("rForearmTwist"), FVector(-14.2, 0, -0.3)); + + AddBone(TEXT("rCarpal1"), TEXT("rHand"), FVector(-0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("rIndex1"), FVector(-7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("rIndex2"), FVector(-3.9, 0, 0)); + AddBoneToLast(TEXT("rIndex3"), FVector(-2.1, 0, 0)); + AddBone(TEXT("rCarpal2"), TEXT("rHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("rMid1"), FVector(-7.5, -0.3, 0)); + AddBoneToLast(TEXT("rMid2"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("rMid3"), FVector(-2.5, 0, 0)); + AddBone(TEXT("rCarpal3"), TEXT("rHand"), FVector(-0.8, -0.4, 0.8)); + AddBoneToLast(TEXT("rRing1"), FVector(-6.9, -0.2, 0)); + AddBoneToLast(TEXT("rRing2"), FVector(-4.0, 0, 0)); + AddBoneToLast(TEXT("rRing3"), FVector(-2.2, 0, 0)); + AddBone(TEXT("rCarpal4"), TEXT("rHand"), FVector(-0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("rPinky1"), FVector(-6.5, 0.2, 0)); + AddBoneToLast(TEXT("rPinky2"), FVector(-2.8, 0, 0)); + AddBoneToLast(TEXT("rPinky3"), FVector(-1.7, 0, 0)); + AddBone(TEXT("rThumb1"), TEXT("rHand"), FVector(-1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("rThumb2"), FVector(-4.1, 0, 0)); + AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp new file mode 100644 index 0000000..eec8f4a --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -0,0 +1,387 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIStructs.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +// Utility conversion functions for compact representation and from arrays to vectors + +float UintB64ToUint(char a, char b) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + + return reverse_map[static_cast(a)] * 64 + reverse_map[static_cast(b)]; +} +uint32 UintB64ToUint(char a, char b, char c) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + return reverse_map[static_cast(a)] * 4096 + reverse_map[static_cast(b)] * 64 + reverse_map[static_cast(c)]; +} + +float FixedB64pairToFloat(char a, char b) { + static const float firstByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9384465070835368, 0.9697117733268197, 0.9384465070835368, 0.9384465070835368, 0.9697117733268197, 0.6257938446507083, 0.6570591108939912, 0.688324377137274, 0.7195896433805569, 0.7508549096238397, 0.7821201758671226, 0.8133854421104054, 0.8446507083536883, 0.8759159745969711, 0.907181240840254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, -0.9687347337567171, -0.9374694675134343, -0.9062042012701514, -0.8749389350268686, -0.8436736687835857, -0.8124084025403029, -0.78114313629702, -0.7498778700537372, -0.7186126038104543, -0.6873473375671715, -0.6560820713238886, -0.6248168050806058, -0.5935515388373229, -0.5622862725940401, -0.5310210063507572, -0.49975574010747437, -0.4684904738641915, -0.43722520762090866, -0.4059599413776258, -0.37469467513434296, -0.3434294088910601, -0.31216414264777725, -0.2808988764044944, -0.24963361016121155, -0.2183683439179287, 0.0, 0.0, 0.0, 0.0, 0.9697117733268197, 0.0, -0.18710307767464585, -0.155837811431363, -0.12457254518808014, -0.09330727894479729, -0.06204201270151444, -0.030776746458231585, 0.0004885197850512668, 0.03175378602833412, 0.06301905227161697, 0.09428431851489982, 0.12554958475818268, 0.15681485100146553, 0.18808011724474838, 0.21934538348803123, 0.2506106497313141, 0.28187591597459694, 0.3131411822178798, 0.34440644846116264, 0.3756717147044455, 0.40693698094772834, 0.4382022471910112, 0.46946751343429405, 0.5007327796775769, 0.5319980459208598, 0.5632633121641426, 0.5945285784074255 }; + static const float secondByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.030288226673180263, 0.030776746458231558, 0.030288226673180263, 0.030288226673180263, 0.030776746458231558, 0.025403028822667317, 0.025891548607718612, 0.026380068392769906, 0.0268685881778212, 0.027357107962872496, 0.02784562774792379, 0.028334147532975085, 0.02882266731802638, 0.029311187103077674, 0.02979970688812897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004885197850512946, 0.0009770395701025891, 0.0014655593551538837, 0.0019540791402051783, 0.002442598925256473, 0.0029311187103077674, 0.003419638495359062, 0.0039081582804103565, 0.004396678065461651, 0.004885197850512946, 0.00537371763556424, 0.005862237420615535, 0.006350757205666829, 0.006839276990718124, 0.0073277967757694185, 0.007816316560820713, 0.008304836345872008, 0.008793356130923302, 0.009281875915974597, 0.009770395701025891, 0.010258915486077186, 0.01074743527112848, 0.011235955056179775, 0.01172447484123107, 0.012212994626282364, 0.0, 0.0, 0.0, 0.0, 0.030776746458231558, 0.0, 0.012701514411333659, 0.013190034196384953, 0.013678553981436248, 0.014167073766487542, 0.014655593551538837, 0.015144113336590131, 0.015632633121641426, 0.01612115290669272, 0.016609672691744015, 0.01709819247679531, 0.017586712261846604, 0.0180752320468979, 0.018563751831949193, 0.019052271617000488, 0.019540791402051783, 0.020029311187103077, 0.02051783097215437, 0.021006350757205666, 0.02149487054225696, 0.021983390327308255, 0.02247191011235955, 0.022960429897410845, 0.02344894968246214, 0.023937469467513434, 0.024425989252564728, 0.024914509037616023 }; + return firstByte[static_cast(a)] + secondByte[static_cast(b)]; +} + +void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { + flatArray.Reserve(flatArray.Num() + data.Len() / 2); + for (int i = 0; i + 1 < data.Len(); i += 2) + flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); +} + +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { + quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); + for (int i = 0; i + 3 < flatArray.Num(); i += 4) + quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); +} + + +void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ + fieldVector2D.X = (*value)[0]->AsNumber(); + fieldVector2D.Y = (*value)[1]->AsNumber(); + } +} + +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + +void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { + if (value.Num() >= 2) { + fieldVector2D.X = value[0]; + fieldVector2D.Y = value[1]; + } +} + +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + +void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { + if (newValue != field) + changeFlag = true; + field = newValue; +} +void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { + SetAndCheckForChange(newValue > 0.5f, field, changeFlag); +} +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} + +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} + +FString FPoseAIHandshake::ToString() const { + return FString::Printf( + TEXT("{\"HANDSHAKE\":{" + "\"name\":\"Unreal LiveLink\"," + "\"rig\":\"%s\", " + "\"mode\":\"%s\", " + "\"face\":\"%s\", " + "\"context\":\"%s\", " + "\"whoami\":\"%s\", " + "\"signature\":\"%s\", " + "\"mirror\":\"%s\", " + "\"syncFPS\": %d, " + "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " + "\"packetFormat\": %d" + "}}"), + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), + *whoami, + *signature, + *(YesNoString(isMirrored)), + syncFPS, + cameraFPS, + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) + ); +} + + +bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const +{ + return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; +} + + + +FString FPoseAIModelConfig::ToString() const { + return FString::Printf( + TEXT("{\"CONFIG\":{" + "\"mirror\":\"%s\", " + "\"stepSensitivity\":%f, " + "\"armSensitivity\":%f, " + "\"crouchSensitivity\": %f, " + "\"jumpSensitivity\":%f" + "}}"), + *(YesNoString(isMirrored)), + stepSensitivity, + armSensitivity, + crouchSensitivity, + jumpSensitivity + ); +} + +bool FPoseAIEventPairBase::CheckTriggerAndUpdate() { + bool hasChanged = Count != InternalCount; + InternalCount = Count; + return hasChanged; +} + +void FPoseAIEventPair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Magnitude = FixedB64pairToFloat(compactString[3], compactString[4]); +} + +void FPoseAIGesturePair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Current = UintB64ToUint(compactString[3], compactString[4]); +} + +void FPoseAIEventStruct::ProcessCompactBody(const FString& compactString) { + TArray compactOrder = { &Footstep, &SidestepL, &SidestepR, &Jump, &FeetSplit, &ArmPump, &ArmFlex, &ArmGestureL, &ArmGestureR}; + if (compactString.Len() % 5 != 0) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: Invalid event string: %s."), *compactString); + return; + } + for (int i = 0; i < compactOrder.Num(); ++i) { + if (compactString.Len() < 5 * i + 5) + break; + compactOrder[i]->ProcessCompact(compactString.Mid(i * 5, 5)); + } +} + +void FPoseAIVisibilityFlags::ProcessCompact(const FString& visString) { + hasChanged = false; + SetAndCheckForChange(visString[0] != '0', isTorso, hasChanged); + SetAndCheckForChange(visString[1] != '0', isLeftLeg, hasChanged); + SetAndCheckForChange(visString[2] != '0', isRightLeg, hasChanged); + SetAndCheckForChange(visString[3] != '0', isLeftArm, hasChanged); + SetAndCheckForChange(visString[4] != '0', isRightArm, hasChanged); + if (visString.Len() > 5) + SetAndCheckForChange(visString[5] != '0', isFace, hasChanged); +} + +void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) { + int32 idx = 0; + if(compactString.Len() < 14) return; + bodyHeight = FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) + 1.0f; + chestYaw = FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f; + stanceYaw = FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 180.0f; + stableFeet = UintB64ToUint(compactString[idx + 6], compactString[idx + 7]); + handZoneLeft = UintB64ToUint(compactString[idx + 8], compactString[idx + 9]); + handZoneRight = UintB64ToUint(compactString[idx + 10], compactString[idx + 11]); + isCrouching = UintB64ToUint(compactString[idx + 12], compactString[idx + 13]) > 0; +} + +void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { + //tbd - this could be simplified if we don't need to keep supported older versions of the api + int32 idx = 0; + if (compactString.Len() < 12) return; + upperBodyLean.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 180.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f + ); + idx += 4; + hipScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx+1]), + FixedB64pairToFloat(compactString[idx+2], compactString[idx+3]) + ); + idx += 4; + chestScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]), + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) + ); + idx += 4; + if (compactString.Len() < idx + 12) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + handIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + if (compactString.Len() < idx + 18) return; + rootTranslation.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; +} + +void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessLeftHand = handObj->GetNumberField("Open"); +} + +void FPoseAILiveValues::ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessRightHand = handObj->GetNumberField("Open"); +} + + +void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { + hasChanged = false; + SetAndCheckForChange(scalars.VisTorso, isTorso, hasChanged); + SetAndCheckForChange(scalars.VisLegL, isLeftLeg, hasChanged); + SetAndCheckForChange(scalars.VisLegR, isRightLeg, hasChanged); + SetAndCheckForChange(scalars.VisArmL, isLeftArm, hasChanged); + SetAndCheckForChange(scalars.VisArmR, isRightArm, hasChanged); +} + + +const FString FPoseAILiveValues::fieldPointScreen = FString(TEXT("PointScreen")); +const FString FPoseAILiveValues::fieldThumbScreen = FString(TEXT("ThumbScreen")); + +void FPoseAIScalarStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody) { + FJsonObjectConverter::JsonObjectToUStruct(scaBody.ToSharedRef(), this); +} + +void FPoseAIEventStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody) { + FJsonObjectConverter::JsonObjectToUStruct(eveBody.ToSharedRef(), this); +} + +void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) { + FJsonObjectConverter::JsonObjectToUStruct(jsonObj.ToSharedRef(), this); +} + + +void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ + bodyHeight = verbose.Scalars.BodyHeight; + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); + stanceYaw = verbose.Scalars.StanceYaw * 180.0f; + chestYaw = verbose.Scalars.ChestYaw * 180.0f; + isCrouching = verbose.Scalars.IsCrouching > 0.5f; + handZoneLeft = verbose.Scalars.HandZoneL; + handZoneRight= verbose.Scalars.HandZoneR; + ProcessArrayAsVector2D(verbose.Vectors.HipLean, upperBodyLean); + upperBodyLean *= 180.0f; + ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); + ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); + ProcessArrayAsVector3D(verbose.Vectors.Hip, rootTranslation); + ProcessArrayAsVector3D(verbose.Vectors.FootIkL, footIkL); + ProcessArrayAsVector3D(verbose.Vectors.FootIkR, footIkR); +} + + + +void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); +} + +void FPoseAILiveValues::ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); +} + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp new file mode 100644 index 0000000..70554f9 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -0,0 +1,291 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "SPoseAILiveLinkWidget.h" +#include "PoseAILiveLinkSourceFactory.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +TWeakPtr SPoseAILiveLinkWidget::source = nullptr; + +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking +static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; + +const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; +int32 SPoseAILiveLinkWidget::syncFPS = 60; +int32 SPoseAILiveLinkWidget::cameraFPS = 60; +int32 SPoseAILiveLinkWidget::modeIndex = 0; +int32 SPoseAILiveLinkWidget::rigIndex = 0; +bool SPoseAILiveLinkWidget::isMirrored = false; +bool SPoseAILiveLinkWidget::isIPv6 = false; + +BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION +void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) +{ + GConfig->GetBool(*section, TEXT("isIPv6"), isIPv6, GEditorIni); + GConfig->GetBool(*section, TEXT("isMirrored"), isMirrored, GEditorIni); + + GConfig->GetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + if (modeIndex < 0 || modeIndex >= PoseAI_Modes.Num()) + modeIndex = 0; + GConfig->GetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + if (rigIndex < 0 || rigIndex >= PoseAI_Rigs.Num()) + rigIndex = 0; + + GConfig->GetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + + ChildSlot + [ + SNew(SBox).WidthOverride(300) + [ + SNew(SVerticalBox) + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("UseIPv6", "Connect via an IPv6 socket")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(ipv6CheckBox, SCheckBox) + .IsChecked(isIPv6 ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.75f) + [ + SNew(STextBlock).Text(LOCTEXT("PortIPv4", "Port for IPv4 (0 if unused)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).FillWidth(0.25f) + [ + SAssignNew(portInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdatePort) + .Text(FText::FromString(FString::FromInt(portNum))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.65f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleMode", "Toggle Camera Mode")) + + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.45f) + + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleModeClicked) + [ + SAssignNew(modeInput, STextBlock) + .Text(FText::FromString(PoseAI_Modes[modeIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.6f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleRig", "Toggle rig format")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.4f) + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleRigClicked) + [ + SAssignNew(rigInput, STextBlock) + .Text(FText::FromString(PoseAI_Rigs[rigIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("IsMirrored", "Mirror camera (flip left/right)")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(mirroredCheckBox, SCheckBox) + .IsChecked(isMirrored ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("SyncFPS", "Smoothed FPS (by app)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(syncFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateSyncFPS) + .Text(FText::FromString(FString::FromInt(syncFPS))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("CameraFPS", "Request Camera FPS")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(cameraFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateCameraFPS) + .Text(FText::FromString(FString::FromInt(cameraFPS))) + ] + ] + + SVerticalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).AutoHeight() + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnOkClicked) + [ + SNew(STextBlock) + .Text(LOCTEXT("OK", "OK")) + ] + ] + ] + ]; +} +END_SLATE_FUNCTION_BUILD_OPTIMIZATION + + +void SPoseAILiveLinkWidget::UpdatePort(const FText& InText, ETextCommit::Type type) +{ + portNum = FCString::Atoi(*(InText.ToString())); +} + + +bool SPoseAILiveLinkWidget::IsPortValid() const +{ + if (portNum < 1028 || portNum > 49151) { + FLiveLinkLog::Warning(TEXT("PoseAI: %d is an invalid port number. Set a valid port number (>1028 and <49151)."), portNum); + return false; + } + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); + return false; + } + return true; +} + + +void SPoseAILiveLinkWidget::UpdateSyncFPS(const FText& InText, ETextCommit::Type type) +{ + syncFPS = FCString::Atoi(*(InText.ToString())); + if (syncFPS < 0) { + syncFPS = 0; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + +void SPoseAILiveLinkWidget::UpdateCameraFPS(const FText& InText, ETextCommit::Type type) +{ + cameraFPS = FCString::Atoi(*(InText.ToString())); + if (cameraFPS < 24) { + cameraFPS = 24; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + GConfig->SetInt(*section, TEXT("cameraFPS"), cameraFPS, GEditorIni); + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + + +void SPoseAILiveLinkWidget::disableExistingSource() +{ + TSharedPtr linkSource = source.Pin(); + if (linkSource.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); + } +} +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() +{ + FPoseAIHandshake handshake = FPoseAIHandshake(); + handshake.isMirrored = isMirrored; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); + handshake.syncFPS = syncFPS; + handshake.cameraFPS = cameraFPS; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); +} + +FReply SPoseAILiveLinkWidget::OnToggleModeClicked() +{ + modeIndex = (modeIndex+1) % PoseAI_Modes.Num(); + modeInput->SetText(FText::FromString(PoseAI_Modes[modeIndex])); + GConfig->SetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnToggleRigClicked() +{ + rigIndex = (rigIndex + 1) % PoseAI_Rigs.Num(); + rigInput->SetText(FText::FromString(PoseAI_Rigs[rigIndex])); + GConfig->SetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnOkClicked() +{ + ReadCheckBox(mirroredCheckBox, isMirrored); + ReadCheckBox(ipv6CheckBox, isIPv6); + + GConfig->SetBool(*section, TEXT("isMirror"), isMirrored, GEditorIni); + + if (IsPortValid()) { + GConfig->SetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + GConfig->Flush(false, GEditorIni); + FString connectionString = ""; + TSharedPtr src = CreateSource(connectionString); + callback.Execute(src, connectionString); + FLiveLinkLog::Info( + TEXT("PoseAI: Setup source. Rig is in %s format, %s and %s."), + *PoseAI_Rigs[rigIndex], + *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")) + ); + } + return FReply::Handled(); +} + +void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& readTo) +{ + TSharedPtr pin = checkBox.Pin(); + if (pin) + readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h new file mode 100644 index 0000000..977dea1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -0,0 +1,128 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "IPAddress.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "SocketSubsystem.h" + + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port); + + +/** + * Implements a more generic endpoint to allow for both IPv6 and IPv4 networks, based on Epic Games IPv4 endpoint from the Networking module. + * + * Mainly is a wrapper around an FInternetAddr with the helper functions needed to use the networking code with only minor modifications. + * + */ +struct FPoseAIEndpoint +{ + /** Holds the endpoint's IP address. */ + TSharedPtr Address; + + /** Holds the endpoint's port number. */ + uint16 Port; + +public: + + /** Default constructor. */ + FPoseAIEndpoint() { } + + + /** + * Creates and initializes a new endpoint from a given FInternetAddr object. + * + * Note: this constructor will be removed after the socket subsystem has been refactored. + * + * @param InternetAddr The Internet address. + */ + + + FPoseAIEndpoint(const TSharedPtr& InternetAddr) + { + check(InternetAddr.IsValid()); + + int32 OutPort; + Address = InternetAddr; + InternetAddr->GetPort(OutPort); + Port = OutPort; + } + + bool IsValid() const { return Address != nullptr && Address.IsValid(); } + +public: + + /** + * Compares this endpoint with the given endpoint for equality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are equal, false otherwise. + */ + bool operator==(const FPoseAIEndpoint& Other) const + { + return ((Address == Other.Address)); + } + + /** + * Compares this address with the given endpoint for inequality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are not equal, false otherwise. + */ + bool operator!=(const FPoseAIEndpoint& Other) const + { + return (Address != Other.Address); + } + + +public: + + + /** + * Gets a string representation for this endpoint. + * + * @return String representation. + * @see Parse, ToText + */ + POSEAILIVELINK_API FString ToString() const; + + /** + * Gets the display text representation. + * + * @return Text representation. + * @see ToString + */ + FText ToText() const + { + return FText::FromString(ToString()); + } + + TSharedRef ToInternetAddr() const + { + if (CachedSocketSubsystem == nullptr) + Initialize(); + + check(CachedSocketSubsystem != nullptr && "Networking module not loaded and initialized"); + TSharedRef InternetAddr = CachedSocketSubsystem->CreateInternetAddr(Address->GetProtocolType()); + { + InternetAddr->SetRawIp(Address->GetRawIp()); + InternetAddr->SetPort(Address->GetPort()); + } + + return InternetAddr; + } + +public: + + /** Initializes the IP endpoint functionality. */ + static void Initialize(); + + + +private: + /** ISocketSubsystem::Get() is not thread-safe, so we cache it here. */ + static ISocketSubsystem* CachedSocketSubsystem; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h new file mode 100644 index 0000000..3eecfa0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -0,0 +1,395 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "Async/Async.h" +#include "LiveLinkTypes.h" +#include "PoseAIStructs.h" +#include "PoseAIEventDispatcher.generated.h" + + +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); +DECLARE_MULTICAST_DELEGATE_TwoParams(FPoseAIConfigUpdate, const FLiveLinkSubjectName&, FPoseAIModelConfig); + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAISubjectConnected, const FLiveLinkSubjectName&, SubjectName, bool, isReconnection); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIRegisteredAs, const FLiveLinkSubjectName&, SubjectName, FName, ConnectionName); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIFrameReceived); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIVisibilityChange, const FPoseAIVisibilityFlags&, Flags); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAILiveValuesUpdate, const FPoseAILiveValues&, LiveValues); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootstepEvent, float, height, bool, isLeftFoot); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAISidestepEvent, bool, isLeftStep); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootsplitEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmpumpEvent, float, height); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIArmflexEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmjackEvent,bool, isRising); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIArmflapEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIJumpEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAICrouchEvent, bool, isCrouching); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIHandToZoneEvent, int32, zone); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmGestureEvent, int32, armGesture); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIStationaryEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIResetLivePositionEvent); + + +UCLASS(ClassGroup = (PoseAI)) +class POSEAILIVELINK_API UStepCounter : public UObject +{ + GENERATED_BODY() + +public: + /** Convenience setter for timeout and fadeduration at the same time*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + UStepCounter* SetProperties(float timeoutIn = 0.5f, float fadeDuration = 0.2f); + + /** clears all steps. Optionally fades speed over */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + void Halt(bool fade); + + /** Average step distance in [body width, body height] units per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float DistancePerSecond(); + + /** Average number of steps per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float StepsPerSecond(); + + /** Time since last step in seconds*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float TimeSinceLastStep(); + + /** most recent step distance*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float LastDistance(); + + + /** time since last step when motion begins to slow*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float timeout = 0.5f; + + /** after a next step timeout, the speed is faded to zero over this duration*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float fadeDurationOnTimeout = 0.2f; + + /** total steps registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + int32 totalSteps = 0; + + /** total distance registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + float totalDistance = 0.0f; + + void RegisterStep(float stepDistance); + + UStepCounter(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) + { + + times_.SetNumUninitialized(num_to_track); + heights_.SetNumUninitialized(num_to_track); + } + + +private: + float CheckIfActiveAndFade(); + const int32 num_to_track = 4; + int32 num_ = 0; + int32 tail_ = -1; + + float steps_per_second_ = 0.0f; + float distance_per_second_ = 0.0f; + FDateTime last_time_ = FDateTime::Now(); + TArray times_; + TArray heights_; +}; + + +class UPoseAIEventDispatcher; + +UCLASS(ClassGroup = (PoseAI), meta = (BlueprintSpawnableComponent)) +class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent +{ + GENERATED_BODY() + friend UPoseAIEventDispatcher; + + public: + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple portss (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum=8080, bool isIPv6 = false); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + + /** Sends a message to the connected PoseCamera to reconfigure the model with user settings */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void ChangeModelConfig(FPoseAIModelConfig config); + + /** sends disconnect message to app and closes source, freeing up Port*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(); + + /** sends disconnect message to app but does not clsoe source */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Disconnect(); + + /** Get the LiveLink subject name associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectName() { return subjectName; } + + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void RegisterAsFirstAvailable(); + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + static void SetHandshake(const FPoseAIHandshake& handshake); + + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken = true); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Deregister(); + + /** Sets rig height, which scales root motion, and allows scaling per dimension (i.e. set Y=0 for no motion to/from camera) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ScaleMotion(float RigHeight=170.0f, FVector Scale=FVector(1.0f, 1.0f,1.0f)); + + /** compensates for an active source's camera rotation in the real world */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void SetLiveCameraRotation(float pitch, float yaw = 0.0f, float roll=0.0f); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseAsBaseTranslation(); + + /** Have player stand up straight and face screen as part of configuration */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseToOrientCamera(); + + /** Remove all live root motion (sets scalemotion to zero)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ZeroMotion(); + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FDateTime lastFrameReceived; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAIVisibilityFlags visibilityFlags; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAILiveValues mostRecentValues; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* footsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* leftsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* rightsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* feetsplits; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armpumps ; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflexes; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armjacks; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapL; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapR; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* jumps; + + /********** events ********************/ + + /** when the component succeesfully registered with a connection (i.e. a pose camera joined) */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIRegisteredAs onRegistered; + + /** when any body part visisbility flag changes */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIVisibilityChange onVisibilityChange; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAILiveValuesUpdate onLiveValues; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootstepEvent onFootstep; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootsplitEvent onFeetsplit; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepLeftFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepRightFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmpumpEvent onArmpump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflexEvent onArmflex; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmjackEvent onArmjack; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureLeft; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureRight; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIJumpEvent onJump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAICrouchEvent onCrouch; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIStationaryEvent onStationary; + + void SetLiveValues(FPoseAILiveValues values) { + mostRecentValues = values; + } + + virtual void InitializeComponent() override { + Super::InitializeComponent(); + InitializeObjects(); + } + +private: + FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + + void InitializeObjects(); + +}; + + +/** + * Dispatcher to transmit events to blueprints and c++, can access events from all sources + */ +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAIEventDispatcher : public UObject +{ +GENERATED_BODY() +public: + /** Gets the singleton dispatcher. Use this for binding or to get named components */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + static UPoseAIEventDispatcher* GetDispatcher() { + if (theInstance==nullptr) { + theInstance = NewObject(); + theInstance->AddToRoot(); + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: Creating EventDispatcher.")); + } + return theInstance; + } + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void SetHandshake(const FPoseAIHandshake& handshake); + + FPoseAIHandshakeUpdate handshakeUpdate; + FPoseAIConfigUpdate modelConfigUpdate; + FPoseAIDisconnect disconnect; + FPoseAIDisconnect closeSource; + + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + + /** Convenience event to tell animation blueprin*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + void BroadcastResetLivePosition(); + + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISubjectConnected subjectConnected; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIResetLivePositionEvent resetLivePositionEvent; + + // Connection driven events + void BroadcastCloseSource(const FLiveLinkSubjectName& subjectName); + void BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config); + void BroadcastDisconnect(const FLiveLinkSubjectName& subjectName); + void BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName); + void BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName); + + // Pose Camera driven events + void BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight); + void BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising); + void BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching); + void BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep); + void BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastJumps(const FLiveLinkSubjectName& subjectName); + void BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values); + void BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastStationary(const FLiveLinkSubjectName& subjectName); + void BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags); + + bool RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken); + void RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component); + + bool HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component); + + void BroadcastResetZeroLivePosition(); + +private: + static UPoseAIEventDispatcher* theInstance; + const double timeoutInSeconds = 60.0; + TQueue componentQueue; + UPROPERTY() + TMap componentsByName; + TMap knownConnectionsWithTime; + UPoseAIEventDispatcher() : UObject() {}; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h new file mode 100644 index 0000000..75d849b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..13d5dee --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,105 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..56ac13d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,72 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h new file mode 100644 index 0000000..6fd81a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -0,0 +1,144 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAILiveLinkServer.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + + +struct POSEAILIVELINK_API PoseAIPortRecord { + FGuid source; + FName connectionName; + FLiveLinkSubjectKey subjectKey; +}; + +class PoseAILiveLinkSingleSourceListener; + + +/** + * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. + * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource +{ +public: + + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {} + + // custom methods + static bool GetPortGuid(int32 port, FGuid& fguid); + static bool IsValidPort(int32 port); + static FName GetConnectionName(int32 port); + static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); + static FName SubjectNameFromPort(int32 port); + + void disable(); + FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; + int32 port; + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; + + void AddSubject(); + +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h new file mode 100644 index 0000000..1819982 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -0,0 +1,37 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once +#include "CoreMinimal.h" +#include "LiveLinkRetargetAsset.h" +#include "PoseAILiveLinkRetargetRotations.generated.h" + +// Rretarget asset for data coming from Live Link. Remaps rotations onto all bones and only translations for root and pelvis/hip. +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAILiveLinkRetargetRotations : public ULiveLinkRetargetAsset +{ + GENERATED_UCLASS_BODY() + + virtual ~UPoseAILiveLinkRetargetRotations() {} + + //~ Begin UObject Interface + virtual void BeginDestroy() override; + //~ End UObject Interface + + //~ Begin ULiveLinkRetargetAsset interface + virtual void BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) override; + //~ End ULiveLinkRetargetAsset interface + + // allow user to scale translations for differences in skeleton sizes + UPROPERTY(EditAnywhere, Category = Settings) + float scaleTranslation = 1.0f; + +private: + + void OnBlueprintClassCompiled(UBlueprint* TargetBlueprint); + + +#if WITH_EDITOR + /** Blueprint.OnCompiled delegate handle */ + FDelegateHandle OnBlueprintCompiledDelegate; +#endif +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h new file mode 100644 index 0000000..c6d53c7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -0,0 +1,217 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Runtime/Networking/Public/Networking.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "Runtime/Sockets/Public/SocketSubsystem.h" +#include "HAL/RunnableThread.h" +#include "LiveLinkLog.h" +#include "SocketTypes.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "IPAddress.h" +#include "Json.h" +#include "PoseAIStructs.h" +#include "PoseAIUdpSocketReceiver.h" +#include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" + + +class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; +class FPoseAISocketSender; + +// The networking class needs to be rewritten + +class POSEAILIVELINK_API PoseAILiveLinkServer +{ +public: + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); + + ~PoseAILiveLinkServer() { + CleanUp(); + } + + // utility function that identifies host IPv4 address, to be printed in LiveLink console to help user connect to correct address + static bool GetIP(FString& myIP); + + void CleanUp(); + void Disconnect(); + + TSharedPtr GetSocket() const { return serverSocket; } + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; + void SendHandshake() const; + void SetHandshake(const FPoseAIHandshake& handshake); + // receiver will be set on a runnable thread and set once started + void SetReceiver(TSharedPtr receiver) { udpSocketReceiver = receiver; } + + +private: + const static FString fieldPrettyName; + const static FString fieldVersion; + const static FString fieldUUID; + const static FString fieldRigType; + const static FString requiredMinVersion; + + TSharedPtr listener; + TWeakPtr source_; + FPoseAIHandshake handshake; + FName protocolType; + int32 port; + bool cleaningUp = false; + + // time of last connection. After timeout seconds a newer connection can takeover the port. + FDateTime lastConnection; + const double TIMEOUT_SECONDS = 10.0; + + TSharedPtr serverSocket; + + //used to launch receiver without slowing main thread + TSharedPtr poseAILiveLinkRunnable; + + //Listens for packets + TSharedPtr udpSocketReceiver; + + //sends instructions to paired app + TSharedPtr udpSocketSender; + FPoseAIEndpoint endpoint; + + // disconnect message formatted for Pose AI mobile app + FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); + + void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + + + bool HasValidConnection() const; + + FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; + + // make sure mobile app is sufficiently advanced version as both endpoints of software evolve + bool CheckAppVersion(FString version) const; + + //split clean up routine by component + void CleanUpReceiver(); + void CleanUpSender(); + void CleanUpSocket(); + +}; + + + +class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable +{ +public: + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { + myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); + thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); + } + virtual uint32 Run() override; + +protected: + FString myName; + int32 port; + FRunnableThread* thread = nullptr; +private: + PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; + TSharedPtr udpSocketReceiver; +}; + + + + +// built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update +class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable +{ +public: + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { + thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); + } + + + virtual uint32 Run() override { + while (running && thread == nullptr) { + FPlatformProcess::Sleep(0.2); + } + + while (running ) { + Sleep(true); + while (running && sleeping) { + FPlatformProcess::Sleep(0.005); + } + + } + + thread = nullptr; + return 0; + } + + virtual void Stop() override { + running = false; + if (thread != nullptr) { + Sleep(false); + } + } + + bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) + { + if (running) { + + int32 sent = 0; + if (!Socket) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); + return false; + } + + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); + + if (sent != Data->Num()) + return false; + + Sleep(false); + return true; + } + return false; + } + + void Sleep(bool sleep) { + sleeping = sleep; + if (thread != nullptr) + thread->Suspend(sleep); + } + +protected: + /** The network socket. */ + TSharedPtr Socket; + + /** The thread object. */ + FRunnableThread* thread = nullptr; + + bool running = true; + bool sleeping = false; +}; + + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h new file mode 100644 index 0000000..73f462b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -0,0 +1,30 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "LiveLinkSourceFactory.h" +#include "ILiveLinkSource.h" +#include "PoseAILiveLinkSourceFactory.generated.h" + +/** + * + */ +UCLASS() +class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFactory +{ + GENERATED_BODY() + + UPoseAILiveLinkSourceFactory() { + } + + ~UPoseAILiveLinkSourceFactory () { + } + +public: + virtual TSharedPtr< SWidget > BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const override; + virtual TSharedPtr< ILiveLinkSource > CreateSource(const FString& ConnectionString) const override; + virtual FText GetSourceDisplayName() const override; + virtual FText GetSourceTooltip() const override; + virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h new file mode 100644 index 0000000..1e8e6ac --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -0,0 +1,151 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "ILiveLinkSource.h" +#include "LiveLinkSubjectSettings.h" +#include "Roles/LiveLinkAnimationTypes.h" +#include "Json.h" +#include "PoseAIStructs.h" + +struct POSEAILIVELINK_API Remapping +{ + FName TargetJointName; + FQuat RotAdj; + Remapping(FName TargetJointName, FQuat RotAdj) : TargetJointName(TargetJointName), RotAdj(RotAdj) {}; +}; + + + +/** + * Abstract base class for the different rig formats streamable by Pose AI + */ +class POSEAILIVELINK_API PoseAIRig +{ + public: + FLiveLinkStaticDataStruct MakeStaticData(); + bool ProcessFrame(const TSharedPtr, FLiveLinkAnimationFrameData& data); + static bool IsFrameData(const TSharedPtr jsonObject); + static TSharedPtr PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake); + static TWeakPtr GetRigFromSubjectName(const FLiveLinkSubjectName& name); + FName RigType() { return rigType; } + + FPoseAIVisibilityFlags visibilityFlags; + FPoseAILiveValues liveValues; + FPoseAIScalarStruct scalars; + FPoseAIEventStruct events; + + bool useNextRootAsOffset = false; + //ankle to head top height for scaling PoseAI root motion. + float rigHeight = 170.0f; + + float CameraTilt = 0.0f; + + protected: + FLiveLinkStaticDataStruct rig; + FPoseAIVerbose verbose; + static const FString fieldBody; + static const FString fieldRigType; + static const FString fieldHandLeft; + static const FString fieldHandRight; + static const FString fieldRotations; + static const FString fieldEvents; + static const FString fieldScalars; + static const FString fieldVectors; + + protected: + PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake); + virtual ~PoseAIRig() { + }; + + /* sets up skeletal heirarchy and provides default locations for each transform based on default skeleton (i.e. UE4 Mannequen or male Metahuman). + These bone lengths ensure a sensible animation is created even if user does not retarget from livelink */ + virtual void Configure(); //impure for MacOS compatibility + + FLiveLinkSubjectName name; + FName rigType; + + bool includeHands; + bool isMirrored; + bool isLowerBodyRotated; + bool isDesktop; + int32 numBodyJoints = 21; + int32 numHandJoints = 17; + // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) + int32 lowerBodyNumOfJoints = 8; + + int32 rShinJoint = 3; + int32 lShinJoint = 7; + + bool isCrouching = false; + int32 handZoneL = 5; + int32 handZoneR = 5; + int32 stableFeet = 0; + FVector prevRootTranslation = FVector::ZeroVector; + // store translations of deployed rig + TMap boneVectors; + TArray jointNames; + TArray parentIndices; + TArray cachedPose = {}; + + //temporary variable used for convenience in rig construction + FName lastBoneAdded; + + //extra offset for hip bone to accomodate mesh thickness from bone sockets. + float rootHipOffsetZ = 2.0f; + + void AddBone(FName boneName, FName parentName, FVector translation); + void AddBoneToLast(FName boneName, FVector translation); + void CachePose(const TArray& transforms); + void AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + bool ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + bool ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void TriggerEvents(); + void RotateLowerBody180(TArray& quatArray); + + +private: + static TMap> RigMap; +}; + +class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { + public: + PoseAIRigUE4(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { + public: + PoseAIRigMixamo(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + +class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { +public: + PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { +public: + PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h new file mode 100644 index 0000000..32ae5d2 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -0,0 +1,529 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Json.h" +#include "JsonObjectConverter.h" +#include "PoseAIStructs.generated.h" + + + +/* decoding utilities for compact representation */ +float UintB64ToUint(char a, char b); +uint32 UintB64ToUint(char a, char b, char c); +float FixedB64pairToFloat(char a, char b); +void FStringFixed12ToFloat(const FString& data, TArray& flatArray); +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); + +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + + +/* the handshake configures the main parameters of pose camera*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIHandshake +{ + GENERATED_BODY() + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiAppModes mode = EPoseAiAppModes::Room; + + /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; + + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; + + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; + + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString whoami = ""; + + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString signature = ""; + + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + + + bool operator==(const FPoseAIHandshake& Other) const; + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + + +/*adjusts the sensitivity of PoseAI events*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIModelConfig +{ + GENERATED_BODY() + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float stepSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float armSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float crouchSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float jumpSensitivity = 0.5f; + + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + bool isMirrored = true; + + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + +/** base class for the two event notifications formats sent by camera. All events have a uint count, which upon change signifies a new event has been registered + and a second property, either a float or uint. +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /** number of events registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Count = 0; + + virtual void ProcessCompact(const FString& compactString) {}; + bool CheckTriggerAndUpdate(); +private: + uint32 InternalCount = 0; +}; + +/** +*structure to hold a type of event notification +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**magnitude of event where approrpiate */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + float Magnitude = 0.0f; + + void ProcessCompact(const FString& compactString) override; + +}; + + +USTRUCT() +struct POSEAILIVELINK_API FPoseAIGesturePair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**index code for most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Current = 0; + + void ProcessCompact(const FString& compactString) override; + +}; + + +/** +*structure to receive event notifications +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventStruct +{ + GENERATED_BODY() +public: + /** number of footsteps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Footstep; + + /** number of left foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepL; + + /** number of right foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepR; + + /** number of jumps registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Jump; + + /** number of footsplits registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair FeetSplit; + + /** number of arm pumps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmPump; + + /** number of arm flexes registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmFlex; + + /** number of left or dual arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureL; + + /** number of right arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureR; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody); + void ProcessCompactBody(const FString& compactString); + +}; + +/** +*structure to share additional information +*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIScalarStruct +{ + GENERATED_BODY() +public: + /** visibility flags by body part. Correspond to figure in the app */ + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisTorso = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisFace = 0.0f; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 HandZoneL = 5; + + /** location of right hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PosPeAI") + int32 HandZoneR = 5; + + /** Heading in degrees of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float ChestYaw = 0.0f; + + /** Heading in degrees of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float StanceYaw = 0.0f; + + /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float BodyHeight = 0.0f; + + /** whether subject is crouching */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float IsCrouching = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 StableFoot = 0.0f; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody); + +}; + + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors +{ + GENERATED_BODY() +public: + UPROPERTY() + TArray HipLean; + UPROPERTY() + TArray HipScreen; + UPROPERTY() + TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; + UPROPERTY() + TArray Hip; + UPROPERTY() + TArray FootIkL; + UPROPERTY() + TArray FootIkR; +}; + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerbose +{ + GENERATED_BODY() +public: + UPROPERTY() + FPoseAIEventStruct Events; + UPROPERTY() + FPoseAIScalarStruct Scalars; + UPROPERTY() + FPoseAIVerboseBodyVectors Vectors; + void ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj); + +}; + +/** + * structure to store and expose visibility flags for events alerting programmer if subject is out of camera + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVisibilityFlags +{ + GENERATED_BODY() +public: + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isTorso = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isFace = false; + + bool HasChanged() { return hasChanged; } + void ProcessVerbose(FPoseAIScalarStruct& scalars); + void ProcessCompact(const FString& visString); + +private: + bool hasChanged = false; +}; + +/** + * structure to share additional information + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAILiveValues +{ +GENERATED_BODY() +public: + + /** How much subject is leaning in radians, head-to-hips; x to the side, y forward */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D upperBodyLean = FVector2D(0.0f, 0.0f); + + /** estimated stance height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + bool isCrouching = false; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneLeft = 5; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneRight = 5; + + /** location of subject in camera frame, scaled in body units (i.e. multiply by rig height to translate to game world). Pos Y moves toward camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector rootTranslation = FVector(0.0f, -2.0f, 0.0f); + + /** Heading in radians of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float stanceYaw = 0.0f; + + /** Heading in radians of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float chestYaw = 0.0f; + + /** Heading of flattened left foot to right foot vector relative to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 modelLatency = 0.0f; + + /** timestamp according to pose camera device (CMTime), in seconds */ + double timestamp = 0.0; + + /** DEPR current height of jump in body units */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI_DEPR") + float jumpHeight = 0.0f; + + /** DEPR estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI_DEPR") + float bodyHeight = 0.0f; + + /** location of hips in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D hipScreen = FVector2D(0.0f, 0.0f); + + /** location of chest in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D chestScreen = FVector2D(0.0f, 0.0f); + + /** location of left index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandLeft = FVector2D(0.0f, 0.0f); + + /** location of right index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + + /** location of left thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbLeft = FVector2D(0.0f, 0.0f); + + /** location of right thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbRight = FVector2D(0.0f, 0.0f); + + /** how open the left hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessLeftHand = 0.5f; + + /** how open the right hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessRightHand = 0.5f; + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); + + /** if at least one foot has been stationary for a few frames */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 stableFeet = 0; + + FVector rootOffset = FVector(0.0f, 0.0f, 0.0f); + FRotator cameraRotation = FRotator(0.0f, 0.0f, 0.0f); + FVector scaleMotion = FVector(1.0f, 0.0f, 1.0f); + + void ProcessVerboseBody(const FPoseAIVerbose& scalars); + void ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand); + void ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); + void ProcessCompactScalarsBody(const FString& compactString); + void ProcessCompactVectorsBody(const FString& compactString); + void ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject >); + void ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject >); + +private: + static const FString fieldPointScreen; + static const FString fieldThumbScreen; + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h new file mode 100644 index 0000000..9434a67 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -0,0 +1,220 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. +// This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) + +#pragma once + +#include "CoreMinimal.h" +#include "HAL/Runnable.h" +#include "HAL/RunnableThread.h" +#include "Misc/SingleThreadRunnable.h" +#include "Serialization/ArrayReader.h" +#include "Sockets.h" +#include "SocketSubsystem.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" + +#include "PoseAIEndpoint.h" +#include "IPAddress.h" + + + + + +/** + * Temporary fix for concurrency crashes. This whole class will be redesigned. + */ +typedef TSharedPtr FArrayReaderPtr; + +/** + * Delegate type for received data. + * + * The first parameter is the received data. + * The second parameter is sender's IP endpoint. + */ +DECLARE_DELEGATE_TwoParams(FPoseAIOnSocketDataReceived, const FString&, const FPoseAIEndpoint&); //Change delegate name and use our endpoint + + +/** + * Asynchronously receives data from an UDP socket. + */ +class FPoseAIUdpSocketReceiver + : public FRunnable + , private FSingleThreadRunnable +{ +public: + + /** + * Creates and initializes a new socket receiver. + * + * @param InSocket The UDP socket to receive data from. + * @param InWaitTime The amount of time to wait for the socket to be readable. + * @param InThreadName The receiver thread name (for debugging). + */ + FPoseAIUdpSocketReceiver(TSharedPtr InSocket, const FTimespan& InWaitTime, const TCHAR* InThreadName) + : Socket(InSocket) + , Stopping(false) + , Thread(nullptr) + , ThreadName(InThreadName) + , WaitTime(InWaitTime) + { + check(Socket != nullptr); + check(Socket->GetSocketType() == SOCKTYPE_Datagram); + Reader->SetNumUninitialized(MaxReadBufferSize); + SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); + } + + /** Virtual destructor. */ + virtual ~FPoseAIUdpSocketReceiver() + { + if (Thread != nullptr) + { + Thread->Kill(true); + delete Thread; + } + } + +public: + + /** Set the maximum size allocated to read off of the socket. */ + void SetMaxReadBufferSize(uint32 InMaxReadBufferSize) + { + MaxReadBufferSize = InMaxReadBufferSize; + } + + /** Start the receiver thread. */ + void Start() + { + Thread = FRunnableThread::Create(this, *ThreadName, 128 * 1024, TPri_AboveNormal, FPlatformAffinity::GetPoolThreadMask()); + } + + /** + * Returns a delegate that is executed when data has been received. + * + * This delegate must be bound before the receiver thread is started with + * the Start() method. It cannot be unbound while the thread is running. + * + * @return The delegate. + */ + FPoseAIOnSocketDataReceived& OnDataReceived() + { + check(Thread == nullptr); + return DataReceivedDelegate; + } + +public: + + //~ FRunnable interface + + virtual FSingleThreadRunnable* GetSingleThreadInterface() override + { + return this; + } + + virtual bool Init() override + { + return true; + } + + virtual uint32 Run() override + { + while (!Stopping) + { + isUpdating = true; + Update(WaitTime); + isUpdating = false; + } + + return 0; + } + + virtual void Stop() override + { + Stopping = true; + } + + virtual void Exit() override { } + +protected: + + /** Update this socket receiver. */ + void Update(const FTimespan& SocketWaitTime) + { + + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) + { + return; + } + + /************************************************* + Hwere we make changes to specify address with protocol + **********************************************************/ + if (Stopping) + return; + + TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); + uint32 Size; + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) + { + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) + { + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); + DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); + } + + } + + + } + +protected: + + //~ FSingleThreadRunnable interface + + virtual void Tick() override + { + Update(FTimespan::Zero()); + } + +private: + FArrayReaderPtr Reader = MakeShared(true); + /** The network socket. */ + TSharedPtr Socket = nullptr; + + /** Pointer to the socket sub-system. */ + ISocketSubsystem* SocketSubsystem = nullptr; + + /** Flag indicating that the thread is stopping. */ + bool Stopping; + + /** The thread object. */ + FRunnableThread* Thread = nullptr; + + /** The receiver thread's name. */ + FString ThreadName; + + /** The amount of time to wait for inbound packets. */ + FTimespan WaitTime; + + /** The maximum read buffer size used to read the socket. */ + uint32 MaxReadBufferSize = 65507u; + + bool isUpdating = false; + +private: + + /** Holds the data received delegate. */ + FPoseAIOnSocketDataReceived DataReceivedDelegate; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h new file mode 100644 index 0000000..dfa1571 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -0,0 +1,78 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "CoreGlobals.h" +#include "Widgets/SCompoundWidget.h" +#include "Widgets/SBoxPanel.h" +#include "Widgets/Input/SEditableTextBox.h" +#include "Widgets/Text/STextBlock.h" +#include "Widgets/Input/SButton.h" +#include "Widgets/Input/SCheckBox.h" +#include "Widgets/Layout/SBox.h" +#include "Types/WidgetActiveTimerDelegate.h" +#include "SlateOptMacros.h" +#include "Misc/ConfigCacheIni.h" +#include "LiveLinkLog.h" +#include "iLiveLinkSource.h" +#include "LiveLinkSourceFactory.h" +#include "IPAddress.h" +#include "PoseAIStructs.h" + + +/** + * + */ +class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public FWidgetActiveTimerDelegate +{ +public: + SLATE_BEGIN_ARGS(SPoseAILiveLinkWidget) {} + SLATE_END_ARGS() + + /** Constructs this widget with InArgs */ + void Construct(const FArguments& InArgs); + + void setCallback(ULiveLinkSourceFactory::FOnLiveLinkSourceCreated whenCreated) { callback = whenCreated; } + + static void disableExistingSource(); + static TSharedPtr CreateSource(const FString& port); + +protected: + static TWeakPtr source; + ULiveLinkSourceFactory::FOnLiveLinkSourceCreated callback; + + UPROPERTY(EditAnywhere, Config, Category = Custom) + +private: + const static FString section; + static int32 portNum; + static int32 syncFPS; + static int32 cameraFPS; + static int32 modeIndex; + static int32 rigIndex; + static bool isMirrored; + static bool isIPv6; + + static FPoseAIHandshake GetHandshake(); + void UpdatePort(const FText& InText, ETextCommit::Type type); + void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); + void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); + bool IsPortValid() const; + + FReply OnOkClicked(); + FReply OnToggleModeClicked(); + FReply OnToggleRigClicked(); + + TWeakPtr ipv6CheckBox = nullptr; + TSharedPtr portInput = nullptr; + TSharedPtr syncFpsInput = nullptr; + TSharedPtr cameraFpsInput = nullptr; + TSharedPtr modeInput = nullptr; + TSharedPtr rigInput = nullptr; + TWeakPtr mirroredCheckBox = nullptr; + TWeakPtr mixamoCheckBox = nullptr; + TWeakPtr rootMotionCheckBox = nullptr; + + void ReadCheckBox(TWeakPtr& checkBox, bool& readTo); +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs new file mode 100644 index 0000000..e39bf7b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -0,0 +1,60 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLinkEd : ModuleRules +{ + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.3/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/PoseAILiveLink.uplugin b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/PoseAILiveLink.uplugin new file mode 100644 index 0000000..d2542cf --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/PoseAILiveLink.uplugin @@ -0,0 +1,39 @@ +{ + "FileVersion": 3, + "Version": 1, + "VersionName": "3.1.29", + "FriendlyName": "PoseAI LiveLink", + "Description": "LiveLink plugin to stream from the Pose Camera motion capture engine by PoseAI", + "Category": "Animation", + "CreatedBy": "Pose AI Ltd", + "CreatedByURL": "www.pose-ai.com", + "DocsURL": "www.pose-ai.com/unreal-engine-livelink", + "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/3db8f23ad003492eb85a50de31e5bc57", + "SupportURL": "support@pose-ai.com", + "EngineVersion": "5.4.0", + "CanContainContent": false, + "IsBetaVersion": false, + "Installed": true, + "Modules": [ + { + "Name": "PoseAILiveLink", + "Type": "Runtime", + "LoadingPhase": "Default", + "WhitelistPlatforms": [ + "Win64", + "Mac" + ] + }, + { + "Name": "PoseAILiveLinkEd", + "Type": "UncookedOnly", + "LoadingPhase": "Default" + } + ], + "Plugins": [ + { + "Name": "LiveLink", + "Enabled": true + } + ] +} \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Resources/Icon128.png b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Resources/Icon128.png new file mode 100644 index 0000000..a5f1494 Binary files /dev/null and b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Resources/Icon128.png differ diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs new file mode 100644 index 0000000..2e1e945 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/PoseAILiveLink.Build.cs @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLink : ModuleRules +{ + public PoseAILiveLink(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "Projects", + "Networking", + "Sockets", + "LiveLink", + "LiveLinkInterface", + "Json", + "JsonUtilities", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + // some livelink functionality was moved to this module for UE5 so we need to include it in UE5 builds + BuildVersion Version; + if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out Version)) + { + if (Version.MajorVersion == 5) + { + PublicDependencyModuleNames.AddRange(new string[] { "LiveLinkAnimationCore" }); + } + } + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..073d2a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,112 @@ +// Copyright 2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimationRuntime.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "Animation/AnimTrace.h" +#include "Engine/SkeletalMeshSocket.h" +#include "Engine/SkeletalMesh.h" + + +DECLARE_CYCLE_STAT(TEXT("PoseAIGroundPenetration Eval"), STAT_PoseAIGroundPenetration_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIGroundPenetration + +FAnimNode_PoseAIGroundPenetration::FAnimNode_PoseAIGroundPenetration() + +{ + +} + +void FAnimNode_PoseAIGroundPenetration::GatherDebugData(FNodeDebugData& DebugData) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(GatherDebugData) + FString DebugLine = DebugData.GetNodeName(this); + + DebugLine += "("; + AddDebugNodeData(DebugLine); + DebugLine += FString::Printf(TEXT(" Target: %s)"), *BoneToModify.BoneName.ToString()); + DebugData.AddDebugItem(DebugLine); + + ComponentPose.GatherDebugData(DebugData); +} + +void FAnimNode_PoseAIGroundPenetration::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(EvaluateSkeletalControl_AnyThread) + check(OutBoneTransforms.Num() == 0); + + if (BonesToCheck.Num() + SocketsBoneReference.Num() > 0) { + const FBoneContainer& BoneContainer = Output.Pose.GetPose().GetBoneContainer(); + + FCompactPoseBoneIndex CompactPoseBoneToModify = BoneToModify.GetCompactPoseIndex(BoneContainer); + FTransform NewBoneTM = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToModify); + FVector appliedTranslation = FVector::Zero(); + float minZ = 99999999.0; + + for (auto& b : BonesToCheck) + { + FCompactPoseBoneIndex CompactPoseBoneToCheck = b.GetCompactPoseIndex(BoneContainer); + float z = Output.Pose.GetComponentSpaceTransform(CompactPoseBoneToCheck).GetTranslation().Z; + minZ = FMath::Min(minZ, z); + + } + for (int i = 0; i < SocketsBoneReference.Num(); ++i) + { + const FCompactPoseBoneIndex SocketBoneIndex = SocketsBoneReference[i].GetCompactPoseIndex(BoneContainer); + const FTransform SocketTransform = SocketsLocalTransform[i] * Output.Pose.GetComponentSpaceTransform(SocketBoneIndex) ; + float z = SocketTransform.GetTranslation().Z; + minZ = FMath::Min(minZ, z); + } + if (PinToFloor) appliedTranslation.Z = -minZ; + else appliedTranslation.Z += FMath::Max(0.0f, -minZ); + NewBoneTM.AddToTranslation(appliedTranslation); + OutBoneTransforms.Add(FBoneTransform(CompactPoseBoneToModify, NewBoneTM)); + } + TRACE_ANIM_NODE_VALUE(Output, TEXT("Target"), BoneToModify.BoneName); +} + +bool FAnimNode_PoseAIGroundPenetration::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) +{ + for (const auto& b : BonesToCheck) { + if (!b.IsValidToEvaluate(RequiredBones)) + return false; + } + // if both bones are valid + return (BoneToModify.IsValidToEvaluate(RequiredBones)); +} + + +void FAnimNode_PoseAIGroundPenetration::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + BoneToModify.Initialize(RequiredBones); + for (auto& b : BonesToCheck) { + b.Initialize(RequiredBones); + } + + if (USkeletalMesh* SkelMesh = RequiredBones.GetSkeletalMeshAsset()) + { + SocketsBoneReference.Empty(); + SocketsLocalTransform.Empty(); + for (auto& socketName : SocketsToCheck) { + if (const USkeletalMeshSocket* Socket = SkelMesh->FindSocket(socketName)) + { + FBoneReference socketBoneReference(Socket->BoneName); + socketBoneReference.Initialize(RequiredBones); + SocketsLocalTransform.Add(Socket->GetSocketLocalTransform()); + SocketsBoneReference.Add(socketBoneReference); + } + } + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Required bones missing from FAnimNode_PoseAIGroundPenetration")); + + } + + +} + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..6042e8f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/AnimNode_PoseAIHandTarget.cpp @@ -0,0 +1,107 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#include "AnimNode_PoseAIHandTarget.h" +#include "Engine/Engine.h" +#include "AnimationRuntime.h" +#include "TwoBoneIK.h" +#include "AnimationCoreLibrary.h" +#include "Animation/AnimInstanceProxy.h" +#include "SceneManagement.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "MaterialShared.h" +#include "Animation/AnimTrace.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +DECLARE_CYCLE_STAT(TEXT("PoseAIHandTargetIK Eval"), STAT_PoseAIHandTarget_Eval, STATGROUP_Anim); + + +///////////////////////////////////////////////////// +// FAnimNode_PoseAIHandTarget + +FAnimNode_PoseAIHandTarget::FAnimNode_PoseAIHandTarget() + : IKBoneCompactPoseIndex(INDEX_NONE) + , SpineFirstIndex(INDEX_NONE) + , LeftUpperArmIndex(INDEX_NONE) + , RightUpperArmIndex(INDEX_NONE) + , IndexFingerTipIndex(INDEX_NONE) +{ +} + + +void FAnimNode_PoseAIHandTarget::EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) +{ + // we only care in the zone so fade IK as we move outside the zone + const float alphaZone = FMath::Clamp(2.0f - FMath::Max(1.0f, FMath::Max(PoseAiIkVector.Y, PoseAiIkVector.Z)), 0.0f, 1.0f); + if (alphaZone == 0.0f || PoseAiIkVector == FVector::ZeroVector) + return; + + const FVector BaseCSPos = Output.Pose.GetComponentSpaceTransform(IKBoneCompactPoseIndex).GetTranslation(); + const FVector Control1CSPos = Output.Pose.GetComponentSpaceTransform(SpineFirstIndex).GetTranslation(); + const FVector Control2CSPos = Output.Pose.GetComponentSpaceTransform(LeftUpperArmIndex).GetTranslation(); + const FVector Control3CSPos = Output.Pose.GetComponentSpaceTransform(RightUpperArmIndex).GetTranslation(); + const FVector Control4CSPos = Control1CSPos + 0.5f * (FVector::Dist(Control2CSPos, Control1CSPos) + FVector::Dist(Control3CSPos, Control1CSPos)) * + FVector::CrossProduct(Control3CSPos - Control1CSPos, Control2CSPos - Control1CSPos).GetSafeNormal(); + FVector TargetLocation = + PoseAiIkVector.X * Control1CSPos + + PoseAiIkVector.Y * Control2CSPos + + PoseAiIkVector.Z * Control3CSPos + + (1.0f - PoseAiIkVector.X - PoseAiIkVector.Y - PoseAiIkVector.Z) * Control4CSPos; + + /* disabled currently until hand stability improves + // adjust wrist IK target by relative position, as angle will be preserved. + if (IndexFingerTipIndex != INDEX_NONE) { + const FVector IndexCSPos = Output.Pose.GetComponentSpaceTransform(IndexFingerTipIndex).GetTranslation(); + TargetLocation -= (IndexCSPos - BaseCSPos); + } + */ + + EffectorLocation = alphaZone * TargetLocation + (1.0f - alphaZone) * BaseCSPos; + EffectorLocationSpace = BCS_ComponentSpace; + + JointTargetLocationSpace = BCS_ComponentSpace; + JointTargetLocation = Output.Pose.GetComponentSpaceTransform(CachedLowerLimbIndex).GetTranslation(); + Super::EvaluateSkeletalControl_AnyThread(Output, OutBoneTransforms); + +} +bool FAnimNode_PoseAIHandTarget::IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) { + if (SpineFirstIndex == INDEX_NONE || LeftUpperArmIndex == INDEX_NONE || RightUpperArmIndex == INDEX_NONE) + return false; + return Super::IsValidToEvaluate(Skeleton, RequiredBones); +} + + +void FAnimNode_PoseAIHandTarget::InitializeBoneReferences(const FBoneContainer& RequiredBones) +{ + DECLARE_SCOPE_HIERARCHICAL_COUNTER_ANIMNODE(InitializeBoneReferences) + IKBone.Initialize(RequiredBones); + + EffectorTarget.InitializeBoneReferences(RequiredBones); + JointTarget.InitializeBoneReferences(RequiredBones); + + IKBoneCompactPoseIndex = IKBone.GetCompactPoseIndex(RequiredBones); + CachedLowerLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + CachedUpperLimbIndex = FCompactPoseBoneIndex(INDEX_NONE); + if (IKBoneCompactPoseIndex != INDEX_NONE) + { + CachedLowerLimbIndex = RequiredBones.GetParentBoneIndex(IKBoneCompactPoseIndex); + if (CachedLowerLimbIndex != INDEX_NONE) + { + CachedUpperLimbIndex = RequiredBones.GetParentBoneIndex(CachedLowerLimbIndex); + } + } + + SpineFirst.Initialize(RequiredBones); + LeftUpperArm.Initialize(RequiredBones); + RightUpperArm.Initialize(RequiredBones); + + SpineFirstIndex = SpineFirst.GetCompactPoseIndex(RequiredBones); + LeftUpperArmIndex = LeftUpperArm.GetCompactPoseIndex(RequiredBones); + RightUpperArmIndex = RightUpperArm.GetCompactPoseIndex(RequiredBones); + + UseIndexFingerTip.Initialize(RequiredBones); + IndexFingerTipIndex = UseIndexFingerTip.GetCompactPoseIndex(RequiredBones); + +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp new file mode 100644 index 0000000..6a2d82f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEndpoint.cpp @@ -0,0 +1,43 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEndpoint.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +ISocketSubsystem* FPoseAIEndpoint::CachedSocketSubsystem = nullptr; + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port) { + + FName socketType = NAME_DGram; + FSocket* socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(socketType, description, protocolType); + if (!socket->SetNonBlocking(true)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI Could not set socket to non-blocking")); + + } + + socket->SetReuseAddr(); + int actualSize; + socket->SetReceiveBufferSize(64 * 1024, actualSize); + socket->SetSendBufferSize(64 * 1024, actualSize); + + TSharedRef sender = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr(protocolType); + sender->SetIp(0); + sender->SetPort(port); + socket->Bind(*sender); + return MakeShareable(socket); +} + + +FString FPoseAIEndpoint::ToString() const +{ + return Address->ToString(true); +} + + +void FPoseAIEndpoint::Initialize() +{ + CachedSocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp new file mode 100644 index 0000000..440a2df --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIEventDispatcher.cpp @@ -0,0 +1,468 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +void UStepCounter::Halt(bool fade) { + num_ = 0; + tail_ = -1; + if (fade) + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout))); + else { + last_time_ = FMath::Min(last_time_, FDateTime::Now() - FTimespan::FromSeconds(static_cast(timeout + fadeDurationOnTimeout))); + steps_per_second_ = 0.0f; + distance_per_second_ = 0.0f; + } +} + +float UStepCounter::CheckIfActiveAndFade() { + float time_since_last_step = TimeSinceLastStep(); + if (time_since_last_step > timeout) { + num_ = 0; + tail_ = -1; + return (fadeDurationOnTimeout > 0.0f) ? + FMath::Max(0.0f, 1.0f - (time_since_last_step - timeout) / fadeDurationOnTimeout) + : 0.0f; + } + else + return 1.0f; +} + +float UStepCounter::StepsPerSecond() { + return steps_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::DistancePerSecond() { + return distance_per_second_ * CheckIfActiveAndFade(); +} + +float UStepCounter::LastDistance() { + return (num_ > 0) ? heights_[tail_] : 0.0f; +} + +float UStepCounter::TimeSinceLastStep() { + return (FDateTime::Now() - last_time_).GetTotalSeconds();; +} + +void UStepCounter::RegisterStep(float stepDistance) { + totalSteps++; + totalDistance += stepDistance; + tail_ = (tail_ + 1) % num_to_track; + last_time_ = FDateTime::Now(); + times_[tail_] = last_time_; + heights_[tail_] = stepDistance; + num_ = FGenericPlatformMath::Min(num_ + 1, num_to_track); + + float elapsed_time; + if (num_ < 2) + elapsed_time = 1.0f; + else { + int head_ = (tail_ + 1) % num_; + elapsed_time = (times_[tail_] - times_[head_]).GetTotalSeconds(); + if (elapsed_time <= 0.0) elapsed_time = 1.0f; + } + steps_per_second_ = static_cast(times_.Num()) / elapsed_time; + distance_per_second_ = 0.0f; + for (int h = 0; h < num_; ++h) + distance_per_second_ += heights_[h]; + distance_per_second_ /= elapsed_time; + +} +UStepCounter* UStepCounter::SetProperties(float timeoutIn, float fadeDuration) { + timeout = timeoutIn; + fadeDurationOnTimeout = fadeDuration; + return this; +} + + +bool UPoseAIMovementComponent::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, myIP, portNum, isIPv6); +} + +bool UPoseAIMovementComponent::AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum, bool isIPv6) { + InitializeObjects(); + FLiveLinkSubjectName addedSubjectName; + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, addedSubjectName) && + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, addedSubjectName, true); +} + +FLiveLinkSubjectName UPoseAIMovementComponent::GetSubjectFaceName(){ + return FLiveLinkSubjectName((*(FString("Face-") + subjectName.ToString()))); +} + +void UPoseAIMovementComponent::InitializeObjects() { + footsteps = NewObject(); + leftsteps = NewObject()->SetProperties(0.3f, 0.1f); + rightsteps = NewObject()->SetProperties(0.3f, 0.1f); + feetsplits = NewObject(); + armpumps = NewObject(); + armflexes = NewObject(); + armjacks = NewObject()->SetProperties(1.0f, 0.5f); + armflapL = NewObject()->SetProperties(0.5f, 1.5f); + armflapR = NewObject()->SetProperties(0.5f, 1.5f); + jumps = NewObject(); +} + +bool UPoseAIMovementComponent::RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken) { + InitializeObjects(); + bool success = UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, name, siezeIfTaken); + if (success) + onRegistered.Broadcast(name, PoseAILiveLinkNetworkSource::GetConnectionName(name)); + return success; +} + +void UPoseAIMovementComponent::RegisterAsFirstAvailable() { + InitializeObjects(); + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentForFirstAvailableSubject(this); +} + +void UPoseAIMovementComponent::CloseSource() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCloseSource(subjectName); +} + +void UPoseAIMovementComponent::Disconnect() { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastDisconnect(subjectName); +} + +void UPoseAIMovementComponent::Deregister() { + UPoseAIEventDispatcher::GetDispatcher()->RegisterComponentByName(this, FLiveLinkSubjectName(NAME_None) ,true); + subjectName = FLiveLinkSubjectName(NAME_None); +} + +void UPoseAIMovementComponent::ChangeModelConfig(FPoseAIModelConfig config) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastConfigUpdate(subjectName, config); +} + +void UPoseAIMovementComponent::SetHandshake(const FPoseAIHandshake& handshake) { + UPoseAIEventDispatcher::GetDispatcher()->SetHandshake(handshake); +} + +void UPoseAIMovementComponent::ScaleMotion(float RigHeight, FVector Scale) { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->rigHeight = RigHeight; + lockedRig->liveValues.scaleMotion = Scale; + } +} + +void UPoseAIMovementComponent::SetLiveCameraRotation(float pitch, float yaw, float roll){ + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + // order changed due to rotated root bone in UE + lockedRig->liveValues.cameraRotation = FRotator(roll, yaw, pitch); + } +} + + +void UPoseAIMovementComponent::UseCurrentPoseToOrientCamera() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig==nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + float pitch = -lockedRig->liveValues.upperBodyLean.Y; + float yaw = -lockedRig->liveValues.chestYaw; + lockedRig->liveValues.cameraRotation = FRotator(0.0f, yaw, pitch); + } +} +void UPoseAIMovementComponent::UseCurrentPoseAsBaseTranslation() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.rootOffset = lockedRig->liveValues.rootTranslation; + } +} + +void UPoseAIMovementComponent::ZeroMotion() { + TWeakPtr rig = PoseAIRig::GetRigFromSubjectName(subjectName); + if (rig == nullptr) + return; + if (TSharedPtr lockedRig = rig.Pin()) { + lockedRig->liveValues.scaleMotion = FVector3d::Zero(); + } +} + +bool UPoseAIEventDispatcher::AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject) { + portNum = PoseAILiveLinkNetworkSource::portDefault; + while (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + portNum++; + if (portNum > 49151) + return false; + } + return AddSource(handshake, isIPv6, portNum, myIP, subject); +} + +bool UPoseAIEventDispatcher::AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject) { + PoseAILiveLinkServer::GetIP(myIP); + return PoseAILiveLinkNetworkSource::AddSource(handshake, portNum, isIPv6, subject); +} + +void UPoseAIEventDispatcher::CloseSource(FLiveLinkSubjectName subject) { + BroadcastCloseSource(subject); +} + + +bool UPoseAIEventDispatcher::RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Event dispatcher, registering %s"), *(name.ToString())); + LastMovementComponent = component; + UPoseAIMovementComponent* existing_component; + if (HasComponent(name, existing_component)) { + if (!siezeIfTaken) return false; + existing_component->Deregister(); + } + + FName prev = component->GetSubjectName(); + componentsByName.Remove(prev); + if (name.Name != NAME_None) + componentsByName.Emplace(name, component); + component->lastFrameReceived = FDateTime::Now(); + component->subjectName = name; + return true; +} + + +UPoseAIEventDispatcher* UPoseAIEventDispatcher::theInstance = nullptr; + + +void UPoseAIEventDispatcher::RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component) { + FLiveLinkSubjectName firstUnbound = GetFirstUnboundSubject(); + if (firstUnbound.Name != NAME_None) + component->RegisterAs(firstUnbound, true); + else + componentQueue.Enqueue(component); +} + +FLiveLinkSubjectName UPoseAIEventDispatcher::GetFirstUnboundSubject(bool excludeIdleSubjects) { + for (auto& elem : knownConnectionsWithTime) { + if (excludeIdleSubjects && (FDateTime::Now() - elem.Value).GetTotalSeconds() > timeoutInSeconds) continue; + UPoseAIMovementComponent* component; + if (HasComponent(elem.Key, component) && component->GetSubjectName() == elem.Key) continue; + return elem.Key; + } + return NAME_None; +} + + +void UPoseAIEventDispatcher::BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName) { + knownConnectionsWithTime.Emplace(subjectName, FDateTime::Now()); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* existing_component; + bool isReconnection = HasComponent(subjectName, existing_component); + if (isReconnection) { + if (existing_component != nullptr && IsValid(existing_component) ) existing_component->onRegistered.Broadcast(subjectName, PoseAILiveLinkNetworkSource::GetConnectionName(subjectName)); + } else if (!componentQueue.IsEmpty()) { + UPoseAIMovementComponent* component; + componentQueue.Dequeue(component); + if (component != nullptr && IsValid(component)) { + component->RegisterAs(subjectName, true); + } + } + subjectConnected.Broadcast(subjectName, isReconnection); + }); +} + +void UPoseAIEventDispatcher::BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config) { + modelConfigUpdate.Broadcast(subjectName, config); +} + +void UPoseAIEventDispatcher::BroadcastDisconnect(const FLiveLinkSubjectName& subjectName) { + disconnect.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastCloseSource(const FLiveLinkSubjectName& subjectName) { + closeSource.Broadcast(subjectName); +} + +void UPoseAIEventDispatcher::BroadcastResetLivePosition() { + resetLivePositionEvent.Broadcast(); +} +void UPoseAIEventDispatcher::SetHandshake(const FPoseAIHandshake& handshake) { + handshakeUpdate.Broadcast(handshake); +} + +void UPoseAIEventDispatcher::BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName) { + FDateTime& nameRef = knownConnectionsWithTime.FindOrAdd(subjectName); + nameRef = FDateTime::Now(); + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->lastFrameReceived = FDateTime::Now(); + }); +} + +void UPoseAIEventDispatcher::BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, visibilityFlags]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onVisibilityChange.Broadcast(visibilityFlags); + component->visibilityFlags = visibilityFlags; + } + }); +} + +void UPoseAIEventDispatcher::BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values){ + AsyncTask(ENamedThreads::GameThread, [this, subjectName, values]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onLiveValues.Broadcast(values); + component->SetLiveValues(values); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->footsteps->RegisterStep(stepHeight); + component->onFootstep.Broadcast(stepHeight, isLeftStep); + } + }); +} +void UPoseAIEventDispatcher::BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->feetsplits->RegisterStep(width); + component->onFeetsplit.Broadcast(width, isExpanding); + } + }); +} +void UPoseAIEventDispatcher::BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, stepHeight]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armpumps->RegisterStep(stepHeight); + component->onArmpump.Broadcast(stepHeight); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float width, bool isExpanding) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, width, isExpanding]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armflexes->RegisterStep(width); + component->onArmflex.Broadcast(width, isExpanding); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isRising]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->armjacks->RegisterStep(0.5f); + component->onArmjack.Broadcast(isRising); + } + }); +} + + +void UPoseAIEventDispatcher::BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepLeftFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isLeftStep]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + ((isLeftStep) ? component->leftsteps : component->rightsteps)->RegisterStep(1.0f); + component->onSidestepRightFoot.Broadcast(isLeftStep); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastJumps(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onJump.Broadcast(); + component->jumps->RegisterStep(1.0f); + } + }); +} + +void UPoseAIEventDispatcher::BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, isCrouching]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onCrouch.Broadcast(isCrouching); + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureLeft.Broadcast(gesture); + if (gesture == 10) { + component->armflapL->RegisterStep(1.0f); + component->onArmflapL.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, gesture]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) { + component->onArmGestureRight.Broadcast(gesture); + if (gesture == 10) { + component->armflapR->RegisterStep(1.0f); + component->onArmflapR.Broadcast(); + } + } + }); +} + +void UPoseAIEventDispatcher::BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneL.Broadcast(zone); + }); +} + + +void UPoseAIEventDispatcher::BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName, zone]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onHandToZoneR.Broadcast(zone); + }); +} + +void UPoseAIEventDispatcher::BroadcastStationary(const FLiveLinkSubjectName& subjectName) { + AsyncTask(ENamedThreads::GameThread, [this, subjectName]() { + UPoseAIMovementComponent* component; + if (HasComponent(subjectName, component)) component->onStationary.Broadcast(); + }); +} + + +bool UPoseAIEventDispatcher::HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component) { + if (!componentsByName.Contains(name)) + return false; + component = componentsByName[name]; + return component != nullptr && IsValid(component); +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp new file mode 100644 index 0000000..fbc882c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLink.cpp @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLink.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + + +void FPoseAILiveLinkModule::StartupModule() +{ + +} + +void FPoseAILiveLinkModule::ShutdownModule() +{ + +} + + + +IMPLEMENT_MODULE(FPoseAILiveLinkModule, PoseAILiveLink) diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp new file mode 100644 index 0000000..004c575 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkFaceSubSource.cpp @@ -0,0 +1,115 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#include "PoseAILiveLinkFaceSubSource.h" +#include "PoseAIStructs.h" +#include "Features/IModularFeatures.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +static FName ParseEnumName(FName EnumName) +{ + const int32 BlendShapeEnumNameLength = 22; + FString EnumString = EnumName.ToString(); + return FName(*EnumString.Right(EnumString.Len() - BlendShapeEnumNameLength)); +} + + +PoseAILiveLinkFaceSubSource::PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient) : liveLinkClient(liveLinkClient) { + + //Update the subject key to match latest one + subjectKey = FLiveLinkSubjectKey(poseSubjectKey.Source, FName(*(FString("Face-") + poseSubjectKey.SubjectName.ToString()))); + //Update property names array + StaticData.PropertyNames.Reset((int32)PoseAIFaceBlendShape::MAX); + + //Iterate through all valid blend shapes to extract names + const UEnum* EnumPtr = StaticEnum(); + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const FName ShapeName = ParseEnumName(EnumPtr->GetNameByValue(Shape)); + StaticData.PropertyNames.Add(ShapeName); + } +} + + +bool PoseAILiveLinkFaceSubSource::AddSubject(FCriticalSection& InSynchObject){ + bool success = false; + if (liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkBasicRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + FScopeLock ScopeLock(&InSynchObject); + + if (liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created face subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct StaticDataStruct(FLiveLinkBaseStaticData::StaticStruct()); + FLiveLinkBaseStaticData* BaseStaticData = StaticDataStruct.Cast(); + BaseStaticData->PropertyNames = StaticData.PropertyNames; + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkBasicRole::StaticClass(), MoveTemp(StaticDataStruct)); + success = true; + + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + } + return success; +} + +bool PoseAILiveLinkFaceSubSource::RequestSubSourceShutdown() +{ + if (liveLinkClient) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient = nullptr; + } + return true; +} + + + +void PoseAILiveLinkFaceSubSource::UpdateFace(TSharedPtr jsonPose) +{ + if (liveLinkClient) { + FLiveLinkFrameDataStruct FrameDataStruct(FLiveLinkBaseFrameData::StaticStruct()); + FLiveLinkBaseFrameData* FrameData = FrameDataStruct.Cast(); + FrameData->WorldTime = FPlatformTime::Seconds(); + //FrameData->MetaData.SceneTime = FrameTime; + + FrameData->PropertyValues.Reserve((int32)PoseAIFaceBlendShape::MAX); + if (jsonPose != nullptr && jsonPose->HasField("Face")) { + uint32 packetFormat = 1; + jsonPose->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 0) { + auto blendShapes = jsonPose->GetArrayField("Face"); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]->AsNumber(); + FrameData->PropertyValues.Add(CurveValue); + } + } + else { + TArray blendShapes; + FString compactFace = jsonPose->GetStringField("Face"); + FStringFixed12ToFloat(compactFace, blendShapes); + // Iterate through all of the blend shapes copying them into the LiveLink data type + for (int32 Shape = 0; Shape < (int32)PoseAIFaceBlendShape::MAX; Shape++) + { + const float CurveValue = blendShapes[Shape]; + FrameData->PropertyValues.Add(CurveValue); + } + } + + + // Share the data locally with the LiveLink client + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(FrameDataStruct)); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp new file mode 100644 index 0000000..31b50b0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNativeSource.cpp @@ -0,0 +1,169 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNativeSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* First use the static method to create a source and add it to the LiveLinkClient. +* The LiveLinkClient must own only shared pointer or UE will crash on cleanup. +* The client will respond with receive client when it is registered. We return a weak ptr +* so the caller can access source if necessary, as the LiveLink system really wants to own the only shared ptr. +*/ +TWeakPtr PoseAILiveLinkNativeSource::AddSource(FName subjectName, const FPoseAIHandshake& handshake) { + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + TSharedPtr PoseAISource = MakeShared(subjectName, handshake); + TWeakPtr weakPtr(PoseAISource); + TSharedPtr Source = StaticCastSharedPtr(PoseAISource); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + return weakPtr; + } + else { + return nullptr; + } +} + + + /* the source is initilized by the static method using the name and the handshake parameters. The name + * governs how the source will appear in the LiveLink UI and how to connect in the LiveLinkPose node in the animation blueprint + */ +PoseAILiveLinkNativeSource::PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake) : + subjectName(subjectName), handshake(handshake), status(LOCTEXT("statusConnecting", "connecting")) +{ + UPoseAIEventDispatcher* dispatcher; + dispatcher = UPoseAIEventDispatcher::GetDispatcher(); +} + +/* + After the source is added to the LiveLinkClient, the LiveLinkClient calls back the source. We store the assigned guid and client pointer + and here we add the subject since we will only have one per client +*/ +void PoseAILiveLinkNativeSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + status = FText::FormatOrdered(LOCTEXT("statusLocalConnected", "Connected to {0}"), FText::FromName(subjectName)); + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, subjectName); + liveLinkClient = InClient; + + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); +} + +/* + After the source is added to the LiveLinkClient and does the ReceiveClient callback, the client creates settings and calls this function. +*/ +void PoseAILiveLinkNativeSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + + +bool PoseAILiveLinkNativeSource::AddSubject(){ + + rig = PoseAIRig::PoseAIRigFactory(subjectName, handshake); + + if (rig.IsValid() && liveLinkClient && IsInGameThread()) { + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(subjectKey.SubjectName); + FScopeLock ScopeLock(&InSynchObject); + + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + return false; + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + return true; + } + } + else { + return false; + } + +} + + + +bool PoseAILiveLinkNativeSource::IsSourceStillValid() const { return true; } + + +void PoseAILiveLinkNativeSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + + + +bool PoseAILiveLinkNativeSource::RequestSourceShutdown() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkLocalSource request source shutdown")); + if (liveLinkClient) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + + return true; +} + +void PoseAILiveLinkNativeSource::ReceivePacket(const FString& recvMessage) { + static const FGuid GUID_Error = FGuid(); + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName("PoseAINativeSource")); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from local posecam, %s"), *Reader->GetErrorMessage()); + return; + } + UpdatePose(jsonObject); +} + + +void PoseAILiveLinkNativeSource::UpdatePose(TSharedPtr jsonPose) +{ + + if (liveLinkClient && rig && rig.IsValid()) { + + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(subjectKey.SubjectName); + faceSubSource->UpdateFace(jsonPose); + } + } +} + +FText PoseAILiveLinkNativeSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI mobile"); +} +FText PoseAILiveLinkNativeSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine");; +} + + + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp new file mode 100644 index 0000000..f3dd836 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkNetworkSource.cpp @@ -0,0 +1,237 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkNetworkSource.h" +#include "Features/IModularFeatures.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +/* +* Static method for creating and adding a networked source to LiveLink, as alternative to the menu UI. +*/ +bool PoseAILiveLinkNetworkSource::AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName) { + + if (IModularFeatures::Get().IsModularFeatureAvailable(ILiveLinkClient::ModularFeatureName)) + { + FLiveLinkClient& LiveLinkClient = IModularFeatures::Get().GetModularFeature(ILiveLinkClient::ModularFeatureName); + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Port %d already assigned to another source. Cancelling"), portNum); + FGuid existingSource; + if (PoseAILiveLinkNetworkSource::GetPortGuid(portNum, existingSource)) + LiveLinkClient.RemoveSource(existingSource); + } + TSharedPtr Source = MakeSource(handshake, portNum, isIPv6); + LiveLinkClient.AddSource(Source); + LiveLinkClient.Tick(); + subjectName = SubjectNameFromPort(portNum); + return true; + } + else { + return false; + } +} + +/* +* Factory method to set up smart pointers and link the udp server weakly to its owner. + * The resulting pointer then needs to be added by the caller to the LiveLink. + * To avoid crashes, only LiveLinkClient should have non-weak references to the source + */ +TSharedPtr PoseAILiveLinkNetworkSource::MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6) { + TSharedPtr PoseAISource = MakeShared(handshake, portNum, isIPv6); + TWeakPtr weakSource(PoseAISource); + PoseAISource->udpServer.SetSource(weakSource); + return StaticCastSharedPtr(PoseAISource); +} + +/* +* Should only be wrapped in a smart pointer and created by MakeSource. + */ +PoseAILiveLinkNetworkSource::PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6) : + listener(MakeShared(this)), + udpServer(PoseAILiveLinkServer(handshake, useIPv6, port)), + handshake(handshake), + port(port), + status(LOCTEXT("statusConnecting", "connecting")) +{ + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + + UE_LOG(LogTemp, Display, TEXT("PoseAI: connecting to %d"), port); + + UPoseAIEventDispatcher* dispatcher = UPoseAIEventDispatcher::GetDispatcher(); + dispatcher->handshakeUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SetHandshake); + dispatcher->modelConfigUpdate.AddSP(listener, &PoseAILiveLinkSingleSourceListener::SendConfig); + dispatcher->disconnect.AddSP(listener, &PoseAILiveLinkSingleSourceListener::DisconnectTarget); + dispatcher->closeSource.AddSP(listener, &PoseAILiveLinkSingleSourceListener::CloseTarget); + if (useIPv6) { + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on IPv6 local-link Port:{1}"), FText::FromString(FString::FromInt(port))); + } + else { + FString myIP; + udpServer.GetIP(myIP); + status = FText::FormatOrdered(LOCTEXT("statusConnected", "listening on {0} Port:{1}"), FText::FromString(myIP), FText::FromString(FString::FromInt(port))); + } +} + + +/* +* This method is called by the livelink client when the source has been submitted. At this point the Guid and client have been asigned +*/ +void PoseAILiveLinkNetworkSource::ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) +{ + sourceGuid = InSourceGuid; + subjectKey = FLiveLinkSubjectKey(sourceGuid, SubjectNameFromPort(port)); + PoseAIPortRecord record = PoseAIPortRecord(); + record.source = InSourceGuid; + record.subjectKey = subjectKey; + usedPorts.Add(port, record); + liveLinkClient = InClient; + + AddSubject(); + faceSubSource = TUniquePtr(new PoseAILiveLinkFaceSubSource(subjectKey, liveLinkClient)); + faceSubSource->AddSubject(InSynchObject); + +} + +/* +* This method is called by the LiveLink client after the source has been submitted and after the receiveclient call. +* Still to be confirmed if any call needs to be made to apply the changes we make here to the actual livelink system +*/ +void PoseAILiveLinkNetworkSource::InitializeSettings(ULiveLinkSourceSettings* Settings) { + Settings->BufferSettings.MaxNumberOfFrameToBuffered = 1; + Settings->Mode = ELiveLinkSourceMode::Latest; +} + + +/* +* Once the source is setup and received we can add subjects. Here we also create the rig that corresponds to the subject. We will only have one subject per source +*/ +void PoseAILiveLinkNetworkSource::AddSubject(){ + rig = PoseAIRig::PoseAIRigFactory(SubjectNameFromPort(port), handshake); + if (!rig) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create rig %s"), *handshake.GetRigString()); + return; + } + check(IsInGameThread()); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + FLiveLinkSubjectPreset subject; + subject.bEnabled = true; + subject.Key = subjectKey; + subject.Role = TSubclassOf(ULiveLinkAnimationRole::StaticClass()); + subject.Settings = nullptr; + subject.VirtualSubject = nullptr; + + FScopeLock ScopeLock(&InSynchObject); + if (!liveLinkClient->CreateSubject(subject)) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to create subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: created subject %s"), *(subjectKey.SubjectName.Name.ToString())); + FLiveLinkStaticDataStruct rigDefinition = rig->MakeStaticData(); + liveLinkClient->PushSubjectStaticData_AnyThread(subject.Key, ULiveLinkAnimationRole::StaticClass(), MoveTemp(rigDefinition)); + } +} + + +/* +* The main processing function. For this source the update is called by the udpclient when it receives a frame. +*/ +void PoseAILiveLinkNetworkSource::UpdatePose(TSharedPtr jsonPose) +{ + if (!liveLinkClient ||!rig || !rig.IsValid()) { + return; + } + FLiveLinkFrameDataStruct frameData(FLiveLinkAnimationFrameData::StaticStruct()); + FLiveLinkAnimationFrameData& data = *frameData.Cast(); + data.Transforms.Reserve(100); + if (rig->ProcessFrame(jsonPose, data)) { + liveLinkClient->PushSubjectFrameData_AnyThread(subjectKey, MoveTemp(frameData)); + faceSubSource->UpdateFace(jsonPose); + } + else { + static const FName NAME_JsonError = "PoseAILiveLink_ProcessFrameError"; + FLiveLinkLog::WarningOnce(NAME_JsonError, subjectKey, TEXT("PoseAI: Error processing frame (for instance, rig type mismatch)")); + } +} + + + +void PoseAILiveLinkNetworkSource::SetHandshake(const FPoseAIHandshake& newHandshake) { + bool dirty = handshake != newHandshake; + bool rigChange = handshake.rig != newHandshake.rig; + handshake = newHandshake; + if (rigChange) { + AddSubject(); + } + if (dirty) + udpServer.SetHandshake(handshake); +} + + +bool PoseAILiveLinkNetworkSource::GetPortGuid(int32 port, FGuid& fguid) { + bool has = usedPorts.Contains(port); + if (has) + fguid = usedPorts[port].source; + return has; +} + +FName PoseAILiveLinkNetworkSource::SubjectNameFromPort(int32 port) { + FString NewString = FString("PoseCam@port:") + FString::FromInt(port); + return FName(*NewString); + +} + +void PoseAILiveLinkNetworkSource::SetConnectionName(FName name) { + if (usedPorts.Contains(port)) + usedPorts[port].connectionName = name; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(int32 port) { + return (usedPorts.Contains(port)) ? usedPorts[port].connectionName : NAME_None; +} + +FName PoseAILiveLinkNetworkSource::GetConnectionName(const FLiveLinkSubjectName& name) { + for (const auto& elem : usedPorts) { + if (elem.Value.subjectKey.SubjectName == name) + return elem.Value.connectionName; + } + return NAME_None; +} + +bool PoseAILiveLinkNetworkSource::IsSourceStillValid() const { return true; } + +bool PoseAILiveLinkNetworkSource::IsValidPort(int32 port) { return !usedPorts.Contains(port); } + +void PoseAILiveLinkNetworkSource::disable() +{ + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling the source")); + status = LOCTEXT("statusDisabled", "disabled"); + liveLinkClient = nullptr; +} + +bool PoseAILiveLinkNetworkSource::RequestSourceShutdown() +{ + usedPorts.Remove(port); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: PoseAILiveLinkNetworkSource on port %d closed"), port); + if (liveLinkClient != nullptr) { + faceSubSource->RequestSubSourceShutdown(); + liveLinkClient->RemoveSubject_AnyThread(subjectKey); + liveLinkClient->RemoveSource(sourceGuid); + liveLinkClient = nullptr; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: removing subject %s"), *(subjectKey.SubjectName.Name.ToString())); + } + return true; +} + +FText PoseAILiveLinkNetworkSource::GetSourceType() const { + return LOCTEXT("SourceType", "PoseAI Local"); +} + +FText PoseAILiveLinkNetworkSource::GetSourceMachineName() const { + return LOCTEXT("SourceMachineName", "Unreal Engine Local");; +} + +TMap PoseAILiveLinkNetworkSource::usedPorts = {}; + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp new file mode 100644 index 0000000..7c6b37e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkRetargetRotations.cpp @@ -0,0 +1,82 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkRetargetRotations.h" + +#include "BonePose.h" +#include "BoneContainer.h" +#include "Engine/Blueprint.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "LiveLinkTypes.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "Roles/LiveLinkAnimationTypes.h" + + +UPoseAILiveLinkRetargetRotations::UPoseAILiveLinkRetargetRotations(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +#if WITH_EDITOR + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + if (Blueprint) + { + OnBlueprintCompiledDelegate = Blueprint->OnCompiled().AddUObject(this, &UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled); + } +#endif +} + +void UPoseAILiveLinkRetargetRotations::BeginDestroy() +{ +#if WITH_EDITOR + if (OnBlueprintCompiledDelegate.IsValid()) + { + UBlueprint* Blueprint = Cast(GetClass()->ClassGeneratedBy); + check(Blueprint); + Blueprint->OnCompiled().Remove(OnBlueprintCompiledDelegate); + OnBlueprintCompiledDelegate.Reset(); + } +#endif + + Super::BeginDestroy(); +} + +void UPoseAILiveLinkRetargetRotations::OnBlueprintClassCompiled(UBlueprint* TargetBlueprint) +{ + +} + + +void UPoseAILiveLinkRetargetRotations::BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) +{ + auto rootTransform = InFrameData->Transforms[0]; + auto rootTranslation = rootTransform.GetTranslation() * scaleTranslation; + auto rootZ = FVector3d(0.0f, 0.0f, rootTranslation.Z); + auto rootXY = FVector3d(rootTranslation.X, rootTranslation.Y, 0.0f); + for (int32 i = 0; i < InSkeletonData->BoneNames.Num(); i++) + { + FName boneName = InSkeletonData->BoneNames[i]; + auto jointTransform = InFrameData->Transforms[i]; + const int32 meshIndex = OutPose.GetBoneContainer().GetPoseBoneIndexForBoneName(boneName); + if (meshIndex != INDEX_NONE) + { + FCompactPoseBoneIndex boneIndex = OutPose.GetBoneContainer().MakeCompactPoseIndex(FMeshPoseBoneIndex(meshIndex)); + if (i == 0) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootXY); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + + else if (i == 1) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetTranslation() + rootZ); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + else if (boneIndex != INDEX_NONE) + { + jointTransform.SetLocation(OutPose.GetRefPose(boneIndex).GetLocation()); + jointTransform.SetScale3D(OutPose.GetRefPose(boneIndex).GetScale3D()); + OutPose[boneIndex] = jointTransform; + } + } + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp new file mode 100644 index 0000000..ff8da5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkServer.cpp @@ -0,0 +1,271 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkServer.h" +#include "Async/Async.h" +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +const FString PoseAILiveLinkServer::requiredMinVersion = FString(TEXT("1.2.5")); +const FString PoseAILiveLinkServer::fieldPrettyName = FString(TEXT("userName")); +const FString PoseAILiveLinkServer::fieldUUID = FString(TEXT("UUID")); +const FString PoseAILiveLinkServer::fieldRigType = FString(TEXT("Rig")); +const FString PoseAILiveLinkServer::fieldVersion = FString(TEXT("version")); + + +PoseAILiveLinkServer::PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum) : + listener(MakeShared(this)), + handshake(myHandshake), + port(portNum) +{ + + protocolType = (isIPv6) ? FNetworkProtocolTypes::IPv6 : FNetworkProtocolTypes::IPv4; + + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Creating Server")); + + FString serverName = "PoseAIServerSocketOnPort_" + FString::FromInt(port); + FString senderName = "PoseAILiveLinkSenderOnPort_" + FString::FromInt(port); + serverSocket = BuildUdpSocket(serverName, protocolType, port); + poseAILiveLinkRunnable = MakeShared(port, listener, this); + udpSocketSender = MakeShared(serverSocket, *senderName); + + FString myIP; + if (protocolType == FNetworkProtocolTypes::IPv6) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on IPv6 link-Local address (begins with fe80:) and Port:%d"), port); + } else if (GetIP(myIP)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server on %s Port:%d"), *myIP, port); + } else { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Created Server but can't determine your IP address. You may not have a valid network adapter.")); + } +} + +void PoseAILiveLinkServer::SetSource(TWeakPtr source) { + source_ = source; +} + + +bool PoseAILiveLinkServer::GetIP(FString& myIP) { + bool canBind = false; + TSharedRef localIp = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->GetLocalHostAddr(*GLog, canBind); + + if (localIp->IsValid()) { + myIP = (localIp->ToString(false)); + return true; + } else { + myIP = LOCTEXT("undeterminedIP", "Can't determine your local host IP address").ToString(); + return false; + } +} + +void PoseAILiveLinkServer::CleanUp() { + if (!cleaningUp) { + cleaningUp = true; + CleanUpReceiver(); + CleanUpSender(); + TSharedPtr socketForClosure = serverSocket; + uint32 portnum = port; + // using delayed closure to try to resolve issue where Epic built plugin creates crashes while project plugin doesn't. + Async(EAsyncExecution::Thread, [socketForClosure, portnum]() { + FPlatformProcess::Sleep(2.0); + socketForClosure->Close(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Closed socket on Port:%d"), portnum); + }); + } +} + +void PoseAILiveLinkServer::CleanUpReceiver() { + if (udpSocketReceiver && udpSocketReceiver.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketReceiver")); + udpSocketReceiver->Stop(); + } + + if (poseAILiveLinkRunnable && poseAILiveLinkRunnable.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up serverThread")); + poseAILiveLinkRunnable.Reset(); + } +} +void PoseAILiveLinkServer::CleanUpSender() { + if (udpSocketSender && udpSocketSender.IsValid()) { + Disconnect(); + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Cleaning up socketSender")); + udpSocketSender->Stop(); + udpSocketSender->Exit(); + } +} + + +bool PoseAILiveLinkServer::HasValidConnection() const { + return endpoint.IsValid() && (FDateTime::Now() - lastConnection).GetTotalSeconds() < TIMEOUT_SECONDS; +} + +void PoseAILiveLinkServer::ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + if (cleaningUp) return; + + TSharedPtr jsonObject = MakeShareable(new FJsonObject); + TSharedRef> Reader = TJsonReaderFactory<>::Create(recvMessage); + + if (!FJsonSerializer::Deserialize(Reader, jsonObject)) { + static const FName NAME_JsonError = "PoseAILiveLink_JsonError"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_JsonError, failKey, TEXT("PoseAI: failed to deserialize json object from %s, %s"), *endpointRecv.ToString(), *Reader->GetErrorMessage()); + return; + } + + bool sameAsCurrent = endpoint.IsValid() && (endpoint.ToString() == endpointRecv.ToString()); + if (HasValidConnection() && !sameAsCurrent) { + if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { + endpoint = endpointRecv; //port has changed but IP and phone nmae same so just update endpoint + SendHandshake(); + } + else { //reject + UE_LOG(LogTemp, Display, TEXT("PoseAI: Ignoring contact from %s as already engaged."), *endpointRecv.ToString()); + //consider sending rejected connection a warning message + } + } + else if (!HasValidConnection() && !sameAsCurrent) { // new connection + InitiateConnection(jsonObject, endpointRecv); + + } + else { + if (PoseAIRig::IsFrameData(jsonObject)) { + lastConnection = FDateTime::Now(); + if (source_.IsValid()) { + auto shared_ptr = source_.Pin(); + shared_ptr->UpdatePose(jsonObject); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFrameReceived(shared_ptr->GetSubjectName()); + } + } + else if (ExtractConnectionName(jsonObject, endpointRecv) == PoseAILiveLinkNetworkSource::GetConnectionName(port)) { //is likely a repeat hello message + SendHandshake(); + } + } +} + +void PoseAILiveLinkServer::InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) { + static const FGuid GUID_Error = FGuid(); + FString version; + if (!(jsonObject->TryGetStringField(fieldVersion, version))) { + static const FName NAME_NoAppVersion = "PoseAILiveLink_NoAppVersion"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_NoAppVersion, failKey, TEXT("PoseAI: Incoming connection does not have a hello handshake")); + return; + } + else if (!CheckAppVersion(version)) { + static const FName NAME_AppVersionFail = "PoseAILiveLink_AppVersionFail"; + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_Error, FName(endpointRecv.ToString())); + FLiveLinkLog::WarningOnce(NAME_AppVersionFail, failKey, TEXT("PoseAI: Please update the mobile app to at least version %s"), *requiredMinVersion); + UE_LOG(LogTemp, Error, TEXT("PoseAILiveLink: Unknown app version. Can not safely connect."), *version); + return; + } + FName connectionName = ExtractConnectionName(jsonObject, endpointRecv); + UE_LOG(LogTemp, Display, TEXT("PoseAI: received new contact from %s on port %d"), *(connectionName.ToString()), endpointRecv.Port); + if (source_.IsValid()) { + source_.Pin()->SetConnectionName(connectionName); + endpoint = endpointRecv; + SendHandshake(); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSubjectConnected(source_.Pin()->GetSubjectName()); + lastConnection = FDateTime::Now(); + } + else { + UE_LOG(LogTemp, Warning, TEXT("PoseAI: Unable to setup Source.")); + } +} + +bool PoseAILiveLinkServer::SendString(FString& message) const { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*message); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + return udpSocketSender->Send(bytedata, endpoint); + } + else { + return false; + } +} + + +void PoseAILiveLinkServer::SendHandshake() const { + FString message_string = handshake.ToString(); + if (SendString(message_string)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent handshake %s to %s"), *message_string, *(endpoint.ToString())); + } else { //unsuccessful + static const FName NAME_HandshakeFail = "PoseAILiveLink_HandshakeFail"; + static const FGuid GUID_HandshakeFail = FGuid(); + FLiveLinkSubjectKey failKey = FLiveLinkSubjectKey(GUID_HandshakeFail, FName(endpoint.ToString())); + FLiveLinkLog::WarningOnce(NAME_HandshakeFail, failKey, TEXT("PoseAI: Unable to send the handshake to %s"), *(endpoint.ToString())); + } +} + +void PoseAILiveLinkServer::SetHandshake(const FPoseAIHandshake& newHandshake) { + handshake = newHandshake; + if (endpoint.IsValid()) + SendHandshake(); +} + + + +void PoseAILiveLinkServer::Disconnect() { + if (endpoint.IsValid()) { + FTCHARToUTF8 byteConvert(*disconnect); + TSharedRef, ESPMode::ThreadSafe> bytedata = MakeShared, ESPMode::ThreadSafe>(); + bytedata->Append((uint8*)byteConvert.Get(), byteConvert.Length());; + if (udpSocketSender->Send(bytedata, endpoint)) { + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent disconnect request to %s"), *(endpoint.ToString())); + } + else { //unsuccesful + UE_LOG(LogTemp, Display, TEXT("PoseAI: Unable to disconnect from %s"), *(endpoint.ToString())); + } + } +} + + +FName PoseAILiveLinkServer::ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv) const +{ + FString prettyString; + if (!(jsonObject->TryGetStringField(fieldPrettyName, prettyString))) { + prettyString = "Unknown"; + } + return FName(*(prettyString.Append("@").Append(endpointRecv.Address->ToString(false)))); +} + +bool PoseAILiveLinkServer::CheckAppVersion(FString version) const +{ + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: App version %s vs required version %s."), *version, *requiredMinVersion); + TArray appArray; + version.ParseIntoArray(appArray, TEXT("."), false); + if (appArray.Num() < 3) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: App version %s format error"), *version); + } + + TArray requiredArray; + requiredMinVersion.ParseIntoArray(requiredArray, TEXT("."), false); + + for (int32 i = 0; i < 3; i++) { + int32 app = FCString::Atoi(*appArray[i]); + int32 req = FCString::Atoi(*requiredArray[i]); + if (app < req) + return false; + else if (app > req) + return true; + } + return true; +} + + +uint32 PoseAILiveLinkReceiverRunnable::Run() { + FTimespan inWaitTime = FTimespan::FromMilliseconds(250); + FString receiverName = "PoseAILiveLink_Receiver_On_Port_" + FString::FromInt(port); + udpSocketReceiver = MakeShared(poseAILiveLinkServer->GetSocket(), inWaitTime, *receiverName); + udpSocketReceiver->OnDataReceived().BindSP(listener.ToSharedRef(), &PoseAILiveLinkServerListener::ReceiveUDPDelegate); + udpSocketReceiver->Start(); + poseAILiveLinkServer->SetReceiver(udpSocketReceiver); + poseAILiveLinkServer = nullptr; + listener = nullptr; + thread = nullptr; + return 0; +} + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp new file mode 100644 index 0000000..3eddf9f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAILiveLinkSourceFactory.cpp @@ -0,0 +1,25 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkSourceFactory.h" +#include "SPoseAILiveLinkWidget.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +TSharedPtr UPoseAILiveLinkSourceFactory::BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const +{ + auto rawWidget = SNew(SPoseAILiveLinkWidget); + rawWidget->setCallback(OnLiveLinkSourceCreated); + TSharedPtr myWidget = rawWidget; + return myWidget; +} + +TSharedPtr< ILiveLinkSource > UPoseAILiveLinkSourceFactory::CreateSource(const FString & ConnectionString) const +{ + return SPoseAILiveLinkWidget::CreateSource(ConnectionString); +} + +FText UPoseAILiveLinkSourceFactory::GetSourceDisplayName() const { return LOCTEXT("Pose AI App", "Pose AI App"); } +FText UPoseAILiveLinkSourceFactory::GetSourceTooltip() const { return LOCTEXT("Connect to the Pose AI mobile App", "Connect to the Pose AI mobile App"); } + +#undef LOCTEXT_NAMESPACE \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp new file mode 100644 index 0000000..ef6cf16 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIRig.cpp @@ -0,0 +1,877 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIRig.h" +#include "PoseAIEventDispatcher.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + + +const FString PoseAIRig::fieldBody = FString(TEXT("Body")); +const FString PoseAIRig::fieldRigType = FString(TEXT("Rig")); +const FString PoseAIRig::fieldHandLeft = FString(TEXT("LeftHand")); +const FString PoseAIRig::fieldHandRight = FString(TEXT("RightHand")); +const FString PoseAIRig::fieldRotations = FString(TEXT("Rotations")); +const FString PoseAIRig::fieldScalars = FString(TEXT("Scalars")); +const FString PoseAIRig::fieldEvents = FString(TEXT("Events")); +const FString PoseAIRig::fieldVectors = FString(TEXT("Vectors")); +TMap> PoseAIRig::RigMap = {}; + +bool isDifferentAndSet(int32 newValue, int32& storedValue) { + bool isDifferent = newValue != storedValue; + storedValue = newValue; + return isDifferent; +} + + +PoseAIRig::PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : + name(name), + rigType(FName(handshake.GetRigString())), + includeHands(handshake.IncludesHands()), + isMirrored(handshake.isMirrored), + isLowerBodyRotated(handshake.isLowerBodyRotated), + isDesktop(handshake.mode == EPoseAiAppModes::Desktop) { + Configure(); +} + +TSharedPtr PoseAIRig::PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake) { + TSharedPtr rigPtr; + switch (handshake.rig) { + case EPoseAiRigPresets::MetaHuman: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::Mixamo: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::MixamoAlt: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::DazUE: + rigPtr = MakeShared(name, handshake); + break; + case EPoseAiRigPresets::UE4: + default: + rigPtr = MakeShared(name, handshake);; + break; + } + + rigPtr->Configure(); + RigMap.Add(name, rigPtr); + return rigPtr; +} + +TWeakPtr PoseAIRig::GetRigFromSubjectName(const FLiveLinkSubjectName& name) { + return RigMap.Contains(name)? RigMap[name] : nullptr; +} + + +FLiveLinkStaticDataStruct PoseAIRig::MakeStaticData(){ + FLiveLinkStaticDataStruct staticData; + staticData.InitializeWith(FLiveLinkSkeletonStaticData::StaticStruct(), nullptr); + FLiveLinkSkeletonStaticData* skelData = staticData.Cast(); + check(skelData); + skelData->SetBoneNames(jointNames); + skelData->SetBoneParents(parentIndices); + return staticData; +} + +void PoseAIRig::AddBone(FName boneName, FName parentName, FVector translation) { + jointNames.Emplace(boneName); + if (parentName == boneName) { + parentIndices.Emplace(-1); + } else { + parentIndices.Emplace(jointNames.IndexOfByKey(parentName)); + } + boneVectors.Add(boneName,translation); + lastBoneAdded = boneName; +} + +void PoseAIRig::AddBoneToLast(FName boneName, FVector translation) { + AddBone(boneName, lastBoneAdded, translation); +} + +bool PoseAIRig::IsFrameData(const TSharedPtr jsonObject) +{ + return (jsonObject->HasField(fieldBody)) || (jsonObject->HasField(fieldHandLeft)) || (jsonObject->HasField(fieldHandRight)); +} + +bool PoseAIRig::ProcessFrame(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + + double timestamp; + jsonObject->TryGetNumberField("Timestamp", timestamp); + // drop packets which are older than latest. in case clock changes capping staleness test at 600 seconds. + if (liveValues.timestamp - 600.0 < timestamp && timestamp < liveValues.timestamp) { + return false; + } + liveValues.timestamp = timestamp; + + FString rigStringOut; + if (jsonObject->TryGetStringField(fieldRigType, rigStringOut) && FName(rigStringOut) != rigType) { + static bool not_warned = true; + if (not_warned) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: Rig is streaming in %s format, expected %s format."), *rigStringOut, *rigType.ToString()); + not_warned = false; + } + return false; + } + + uint32 packetFormat = 0; + jsonObject->TryGetNumberField("PF", packetFormat); + + if (packetFormat == 1) + ProcessCompactSupplementaryData(jsonObject, data); + else + ProcessVerboseSupplementaryData(jsonObject, data); + + TriggerEvents(); + + data.WorldTime = FPlatformTime::Seconds(); + bool has_processed = (packetFormat == 1) ? ProcessCompactRotations(jsonObject, data) : ProcessVerboseRotations(jsonObject, data); + return has_processed; +} + +void PoseAIRig::TriggerEvents() { + /* trigger various events and update the Pose AI Movement Component */ + if (visibilityFlags.HasChanged()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastVisibilityChange(name, visibilityFlags); + } + if (verbose.Events.Jump.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastJumps(name); + } + if (verbose.Events.Footstep.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.Footstep.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFootsteps(name, height, verbose.Events.Footstep.Magnitude > 0.0f); + } + if (verbose.Events.FeetSplit.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.FeetSplit.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastFeetsplits(name, width, verbose.Events.FeetSplit.Magnitude < 0.0f); + } + if (verbose.Events.ArmPump.CheckTriggerAndUpdate()) { + float height = FMath::Abs(verbose.Events.ArmPump.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmpumps(name, height); + } + if (verbose.Events.ArmFlex.CheckTriggerAndUpdate()) { + float width = FMath::Abs(verbose.Events.ArmFlex.Magnitude); + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmflexes(name, width, verbose.Events.ArmFlex.Magnitude < 0.0f); + } + if (verbose.Events.SidestepL.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepL(name, verbose.Events.SidestepL.Magnitude < 0.0f); + } + if (verbose.Events.SidestepR.CheckTriggerAndUpdate()) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastSidestepR(name, verbose.Events.SidestepR.Magnitude < 0.0f); + } + if (verbose.Events.ArmGestureL.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureL.Current == 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, true); + else if (verbose.Events.ArmGestureL.Current == 51) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmjacks(name, false); + else + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureL(name, verbose.Events.ArmGestureL.Current); + } + if (verbose.Events.ArmGestureR.CheckTriggerAndUpdate()) { + if (verbose.Events.ArmGestureR.Current < 50) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastArmGestureR(name, verbose.Events.ArmGestureR.Current); + } + if (isDifferentAndSet(liveValues.handZoneLeft, handZoneL)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneL(name, handZoneL); + } + if (isDifferentAndSet(liveValues.handZoneRight, handZoneR)) { + UPoseAIEventDispatcher::GetDispatcher()->BroadcastHandToZoneR(name, handZoneR); + } + if (isDifferentAndSet(liveValues.stableFeet, stableFeet)) { + if (stableFeet > 1) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastStationary(name); + } + if (liveValues.isCrouching != isCrouching) { + isCrouching = !isCrouching; + UPoseAIEventDispatcher::GetDispatcher()->BroadcastCrouches(name, isCrouching); + } + if (visibilityFlags.isTorso) + UPoseAIEventDispatcher::GetDispatcher()->BroadcastLiveValues(name, liveValues); +} + + +void PoseAIRig::ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) { + FString VisA = (objBody->HasTypedField("VisA")) ? objBody->GetStringField("VisA") : ""; + FString ScaA = (objBody->HasTypedField("ScaA")) ? objBody->GetStringField("ScaA") : ""; + FString VecA = (objBody->HasTypedField("VecA")) ? objBody->GetStringField("VecA") : ""; + FString EveA = (objBody->HasTypedField("EveA")) ? objBody->GetStringField("EveA") : ""; + visibilityFlags.ProcessCompact(VisA); + liveValues.ProcessCompactScalarsBody(ScaA); + liveValues.ProcessCompactVectorsBody(VecA); + verbose.Events.ProcessCompactBody(EveA); + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + + } + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + liveValues.ProcessCompactVectorsHandLeft(objHandLeft); + } + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) { + liveValues.ProcessCompactVectorsHandRight(objHandRight); + } +} + +void PoseAIRig::AssignCharacterMotion(FLiveLinkAnimationFrameData& data) { + if (!isDesktop) { + FVector playerMotion = liveValues.cameraRotation.RotateVector(liveValues.rootTranslation - liveValues.rootOffset) * rigHeight * liveValues.scaleMotion; + data.Transforms[0].SetTranslation(playerMotion); + } +} + +bool PoseAIRig::ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + FString rotaBody; + FString rotaHandLeft; + FString rotaHandRight; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotaBody = (objBody->HasTypedField("RotA")) ? objBody->GetStringField("RotA") : ""; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotaHandLeft = (objHandLeft->HasTypedField("RotA")) ? objHandLeft->GetStringField("RotA") : ""; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotaHandRight = (objHandRight->HasTypedField("RotA")) ? objHandRight->GetStringField("RotA") : ""; + + + bool hasProcessedRotations; + + if ((rotaBody.Len() < 8 && cachedPose.Num() < 1) ) { + hasProcessedRotations = false; + } + else if (rotaBody.Len() < 8 ) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + AppendCachedRotations(0, 1, componentRotations, data); + + if (rotaBody.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaBody, flatArray); + FlatArrayToQuats(flatArray, quatArray); + if (isLowerBodyRotated) { + RotateLowerBody180(quatArray); + } + AppendQuatArray(quatArray, 1, componentRotations, data); //start at 1 as pose camera does not include the root joint + } + else + AppendCachedRotations(1, numBodyJoints, componentRotations, data); + + + if (includeHands) { + if (rotaHandLeft.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandLeft, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints, numBodyJoints + numHandJoints, componentRotations, data); + if (rotaHandRight.Len() > 7) { + TArray flatArray; + TArray quatArray; + FStringFixed12ToFloat(rotaHandRight, flatArray); + FlatArrayToQuats(flatArray, quatArray); + AppendQuatArray(quatArray, numBodyJoints + numHandJoints, componentRotations, data); + } + else + AppendCachedRotations(numBodyJoints + numHandJoints, numBodyJoints + 2 * numHandJoints, componentRotations, data); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +bool PoseAIRig::ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > rotBody = nullptr; + TSharedPtr < FJsonObject > rotHandLeft = nullptr; + TSharedPtr < FJsonObject > rotHandRight = nullptr; + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + if (objBody != nullptr && objBody.IsValid()) + rotBody = (objBody->HasTypedField(fieldRotations)) ? objBody->GetObjectField(fieldRotations) : nullptr; + + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + if (objHandLeft != nullptr && objHandLeft.IsValid()) + rotHandLeft = (objHandLeft->HasTypedField(fieldRotations)) ? objHandLeft->GetObjectField(fieldRotations) : nullptr; + + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + if (objHandRight != nullptr && objHandRight.IsValid()) + rotHandRight = (objHandRight->HasTypedField(fieldRotations)) ? objHandRight->GetObjectField(fieldRotations) : nullptr; + + + bool hasProcessedRotations; + if (rotBody == nullptr && cachedPose.Num() < 1) { + hasProcessedRotations = false; + } + else if (rotBody == nullptr || !visibilityFlags.isTorso) { + data.Transforms.Append(cachedPose); + hasProcessedRotations = true; + } + else { + TArray componentRotations; + + for (int32 i = 0; i < jointNames.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + FQuat rotation; + const TArray < TSharedPtr < FJsonValue > >* outArray; + FString jointString = jointName.ToString(); + if ((rotBody != nullptr && rotBody->TryGetArrayField(jointString, outArray)) || + (rotHandLeft != nullptr && rotHandLeft->TryGetArrayField(jointString, outArray)) || + (rotHandRight != nullptr && rotHandRight->TryGetArrayField(jointString, outArray))) { + rotation = FQuat((*outArray)[0]->AsNumber(), (*outArray)[1]->AsNumber(), (*outArray)[2]->AsNumber(), (*outArray)[3]->AsNumber()); + } + else if (cachedPose.Num() > i) { + rotation = parentQuat * cachedPose[i].GetRotation(); + } + else { + rotation = FQuat::Identity; + } + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + //Live link expects local rotations. Consider having this sent directly by PoseAI app to save calculations + data.Transforms.Add(transform); + } + AssignCharacterMotion(data); + CachePose(data.Transforms); + + hasProcessedRotations = true; + } + return hasProcessedRotations; +} + +void PoseAIRig::ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data) +{ + TSharedPtr < FJsonObject > objBody; + TSharedPtr < FJsonObject > objHandLeft; + TSharedPtr < FJsonObject > objHandRight; + TSharedPtr < FJsonObject > scaBody = nullptr; + TSharedPtr < FJsonObject > eveBody = nullptr; + TSharedPtr < FJsonObject > vecBody = nullptr; + TSharedPtr < FJsonObject > vecHandLeft = nullptr; + TSharedPtr < FJsonObject > vecHandRight = nullptr; + + jsonObject->TryGetNumberField("ModelLatency", liveValues.modelLatency); + + objBody = (jsonObject->HasTypedField(fieldBody)) ? jsonObject->GetObjectField(fieldBody) : nullptr; + objHandLeft = (jsonObject->HasTypedField(fieldHandLeft)) ? jsonObject->GetObjectField(fieldHandLeft) : nullptr; + objHandRight = (jsonObject->HasTypedField(fieldHandRight)) ? jsonObject->GetObjectField(fieldHandRight) : nullptr; + + if (objBody != nullptr && objBody.IsValid()) { + verbose.ProcessJsonObject(objBody); + //vecBody = (objBody->HasTypedField(fieldVectors)) ? objBody->GetObjectField(fieldVectors) : nullptr; + } + + liveValues.ProcessVerboseBody(verbose); + + if (objHandLeft != nullptr && objHandLeft.IsValid()) { + vecHandLeft = (objHandLeft->HasTypedField(fieldVectors)) ? objHandLeft->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandLeft(vecHandLeft); + if (objHandLeft->HasTypedField("Open")) + liveValues.opennessLeftHand = objHandLeft->GetNumberField("Open"); + } + if (objHandRight != nullptr && objHandRight.IsValid()) { + vecHandRight = (objHandRight->HasTypedField(fieldVectors)) ? objHandRight->GetObjectField(fieldVectors) : nullptr; + liveValues.ProcessVerboseVectorsHandRight(vecHandRight); + if (objHandRight->HasTypedField("Open")) + liveValues.opennessRightHand = objHandRight->GetNumberField("Open"); + } + + liveValues.jumpHeight = verbose.Events.Jump.Magnitude; + visibilityFlags.ProcessVerbose(verbose.Scalars); +} + +void PoseAIRig::RotateLowerBody180(TArray& quatArray) { + FQuat q180 = FQuat(0.0, 0.0, 1.0, 0.0); + for (int32 i = 0; i < lowerBodyNumOfJoints + 1; ++i) { + quatArray[i] = q180 * quatArray[i]; + } + +} + + +void PoseAIRig::AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < begin + quatArray.Num(); i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + const FQuat& rotation = quatArray[i - begin]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data) { + for (int32 i = begin; i < end; i++) { + const FName& jointName = jointNames[i]; + int32 parentIdx = parentIndices[i]; + FQuat parentQuat = (parentIdx < 0 ? FQuat::Identity : componentRotations[parentIdx]); + const FQuat& rotation = (cachedPose.Num() > i) ? parentQuat * cachedPose[i].GetRotation() : FQuat::Identity; + const FVector& translation = boneVectors.FindRef(jointName); + componentRotations.Add(rotation); + FQuat finalRotation = parentQuat.Inverse() * rotation; + finalRotation.Normalize(); + FTransform transform = FTransform(finalRotation, translation, FVector::OneVector); + data.Transforms.Add(transform); + } +} + +void PoseAIRig::CachePose(const TArray& transforms) { + cachedPose = TArray(transforms); +} + +void PoseAIRig::Configure() {} + +void PoseAIRigUE4::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-1.448829, 0.531424, 9.00581)); + AddBoneToLast(TEXT("calf_r"), FVector(42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(10.453837, -16.577854, 0.080156)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-1.448829, 0.531424, -9.00581)); + AddBoneToLast(TEXT("calf_l"), FVector(-42.572037, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.19669, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-10.453837, 16.577854, 0.080156)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(10.808878, 0.851415, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(18.875349, -3.801159, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(13.407329, -0.420477, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(16.558783, 0.355318, 0)); + AddBoneToLast(TEXT("head"), FVector(9.283613, -0.364157, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_03"), FVector(11.883688, 2.732088, -3.781983)); + AddBoneToLast(TEXT("upperarm_l"), FVector(15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(30.33993, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_03"), FVector(11.883688, 2.732102, 3.782003)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-15.784872, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-30.33993, 0, 0)); + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("index_01_l"), TEXT("hand_l"), FVector(12.068114, -1.763462, -2.109398)); + AddBoneToLast(TEXT("index_02_l"), FVector(4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(3.39379, 0, 0)); + AddBone(TEXT("middle_01_l"), TEXT("hand_l"), FVector(12.244281, -1.293644, 0.571162)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(3.648844, 0, 0)); + AddBone(TEXT("ring_01_l"), TEXT("hand_l"), FVector(11.497885, -1.753527, 2.846912)); + AddBoneToLast(TEXT("ring_02_l"), FVector(4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(3.476652, 0, 0)); + AddBone(TEXT("pinky_01_l"), TEXT("hand_l"), FVector(10.140665, -2.263151, 4.643148)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(2.985631, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(4.762036, -2.374981, -2.53782)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(4.062171, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-26.975143, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("index_01_r"), TEXT("hand_r"), FVector(-12.068114, 1.763462, 2.109398)); + AddBoneToLast(TEXT("index_02_r"), FVector(-4.287498, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-3.39379, 0, 0)); + AddBone(TEXT("middle_01_r"), TEXT("hand_r"), FVector(-12.244281, 1.293644, -0.571162)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.640374, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-3.648844, 0, 0)); + AddBone(TEXT("ring_01_r"), TEXT("hand_r"), FVector(-11.497885, 1.753527, -2.846912)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-4.430177, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-3.476652, 0, 0)); + AddBone(TEXT("pinky_01_r"), TEXT("hand_r"), FVector(-10.140665, 2.263151, -4.643148)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.570981, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-2.985631, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-4.762036, 2.374981, 2.53782)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-3.869672, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-4.062171, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + + rig = MakeStaticData(); +} + + +void PoseAIRigMixamo::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, -44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(-0.7, -17.8, -5.8)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, -44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, -35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0.7, -17.8, -5.8)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(0, -25.7, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("RightForeArm"), FVector(0, -25.7, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(0, -14.0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(0, -23.0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(0, -14.0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigMixamoAlt::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hips"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("RightUpLeg"), TEXT("hips"), FVector(-9.4, 5.0, 0)); + AddBoneToLast(TEXT("RightLeg"), FVector(0, 44.5, 0)); + AddBoneToLast(TEXT("RightFoot"), FVector(0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("RightToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("LeftUpLeg"), TEXT("hips"), FVector(9.4, 5.0, 0)); + AddBoneToLast(TEXT("LeftLeg"), FVector(-0, 44.5, 0)); + AddBoneToLast(TEXT("LeftFoot"), FVector(-0.7, 35.0, -2.4)); + AddBoneToLast(TEXT("LeftToeBase"), FVector(0, 11.0, 13.0)); + + AddBone(TEXT("Spine"), TEXT("hips"), FVector(0, -9.0, -0.3)); + AddBoneToLast(TEXT("Spine1"), FVector(0, -10.5, 0)); + AddBoneToLast(TEXT("Spine2"), FVector(0, -12.0, 0)); + AddBoneToLast(TEXT("Neck"), FVector(0, -13.5, 0)); + AddBoneToLast(TEXT("Head"), FVector(0, -8.2, 2.1)); + + AddBone(TEXT("LeftShoulder"), TEXT("Spine2"), FVector(5.7, -11.8, 0)); + AddBoneToLast(TEXT("LeftArm"), FVector(12.0,0, 0)); + AddBoneToLast(TEXT("LeftForeArm"), FVector(25.7,0, 0)); + + AddBone(TEXT("RightShoulder"), TEXT("Spine2"), FVector(-5.7, -11.8, 0)); + AddBoneToLast(TEXT("RightArm"), FVector(-12.0,0, 0 )); + AddBoneToLast(TEXT("RightForeArm"), FVector(-25.7,0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("LeftHand"), TEXT("LeftForeArm"), FVector(23.0, 0, 0)); + AddBone(TEXT("LeftForeArmTwist"), TEXT("LeftForeArm"), FVector(14.0,0, 0)); + + AddBone(TEXT("LeftHandIndex1"), TEXT("LeftHand"), FVector(-3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("LeftHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("LeftHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("LeftHandMiddle1"), TEXT("LeftHand"), FVector(-0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("LeftHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("LeftHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("LeftHandRing1"), TEXT("LeftHand"), FVector(1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("LeftHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("LeftHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("LeftHandPinky1"), TEXT("LeftHand"), FVector(3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("LeftHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("LeftHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("LeftHandThumb1"), TEXT("LeftHand"), FVector(-2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("LeftHandThumb2"), FVector(-0.7, -3.2, 0)); + AddBoneToLast(TEXT("LeftHandThumb3"), FVector(0.2, -3.0, 0)); + + AddBone(TEXT("RightHand"), TEXT("RightForeArm"), FVector(-23.0,0, 0)); + AddBone(TEXT("RightForeArmTwist"), TEXT("RightForeArm"), FVector(-14.0,0, 0)); + AddBone(TEXT("RightHandIndex1"), TEXT("RightHand"), FVector(3.3, -8.3, 0.1)); + AddBoneToLast(TEXT("RightHandIndex2"), FVector(0, -3.1, 0)); + AddBoneToLast(TEXT("RightHandIndex3"), FVector(0, -2.9, 0)); + AddBone(TEXT("RightHandMiddle1"), TEXT("RightHand"), FVector(0.9, -8.5, -0.1)); + AddBoneToLast(TEXT("RightHandMiddle2"), FVector(0, -3.3, 0)); + AddBoneToLast(TEXT("RightHandMiddle3"), FVector(0, -3.1, 0)); + AddBone(TEXT("RightHandRing1"), TEXT("RightHand"), FVector(-1.1, -8.7, 0.2)); + AddBoneToLast(TEXT("RightHandRing2"), FVector(0, -2.7, 0)); + AddBoneToLast(TEXT("RightHandRing3"), FVector(0, -2.7, 0)); + AddBone(TEXT("RightHandPinky1"), TEXT("RightHand"), FVector(-3.1, -8.0, 0.2)); + AddBoneToLast(TEXT("RightHandPinky2"), FVector(0, -2.5, 0)); + AddBoneToLast(TEXT("RightHandPinky3"), FVector(0, -2.0, 0)); + AddBone(TEXT("RightHandThumb1"), TEXT("RightHand"), FVector(2.9, -2.6, 1.2)); + AddBoneToLast(TEXT("RightHandThumb2"), FVector(0.7, -3.2, 0)); + AddBoneToLast(TEXT("RightHandThumb3"), FVector(-0.2, -3.0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + + + +void PoseAIRigMetaHuman::Configure() +{ + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, 0, 0.0)); + + AddBone(TEXT("thigh_r"), TEXT("pelvis"), FVector(-2.3, 0.4, 9.27)); + AddBoneToLast(TEXT("calf_r"), FVector(41.2, 0, 0)); + AddBoneToLast(TEXT("foot_r"), FVector(40.0, 0, 0)); + AddBoneToLast(TEXT("ball_r"), FVector(7.1, -14.4, -0.4)); + + AddBone(TEXT("thigh_l"), TEXT("pelvis"), FVector(-2.3, 0.4, -9.27)); + AddBoneToLast(TEXT("calf_l"), FVector(-41.2, 0, 0)); + AddBoneToLast(TEXT("foot_l"), FVector(-40.0, 0, 0)); + AddBoneToLast(TEXT("ball_l"), FVector(-7.1, 14.4, 0.4)); + + AddBone(TEXT("spine_01"), TEXT("pelvis"), FVector(3.4, 0.0, 0)); + AddBoneToLast(TEXT("spine_02"), FVector(6.3, 0.0, 0)); + AddBoneToLast(TEXT("spine_03"), FVector(6.9, 0.0, 0)); + AddBoneToLast(TEXT("spine_04"), FVector(8.1, 0.0, 0)); + AddBoneToLast(TEXT("spine_05"), FVector(18.3, 0.0, 0)); + AddBoneToLast(TEXT("neck_01"), FVector(11.6, 1.0, 0)); + AddBoneToLast(TEXT("neck_02"), FVector(5.0, 0, 0)); + AddBoneToLast(TEXT("head"), FVector(5.0, 0, 0)); + + AddBone(TEXT("clavicle_l"), TEXT("spine_05"), FVector(5.5, -0.7, -1.2)); + AddBoneToLast(TEXT("upperarm_l"), FVector(17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_l"), FVector(27.0, 0, 0)); + + AddBone(TEXT("clavicle_r"), TEXT("spine_05"), FVector(5.5, -0.7, 1.2)); + AddBoneToLast(TEXT("upperarm_r"), FVector(-17.0, 0, 0)); + AddBoneToLast(TEXT("lowerarm_r"), FVector(-27.0, 0, 0)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("hand_l"), TEXT("lowerarm_l"), FVector(25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_l"), TEXT("lowerarm_l"), FVector(14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_l"), TEXT("lowerarm_l"), FVector(7.0, 0, 0)); + + AddBone(TEXT("index_metacarpal_l"), TEXT("hand_l"), FVector(3.5, 0.4, -2.1)); + AddBoneToLast(TEXT("index_01_l"), FVector(5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_l"), FVector(2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_l"), TEXT("hand_l"), FVector(3.3, 0.3, -0.1)); + AddBoneToLast(TEXT("middle_01_l"), FVector(6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_l"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_l"), FVector(2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_l"), TEXT("hand_l"), FVector(3.2, -0.2, 1.2)); + AddBoneToLast(TEXT("ring_01_l"), FVector(6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_l"), FVector(3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_l"), FVector(2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_l"), TEXT("hand_l"), FVector(3.1, -0.7, 2.4)); + AddBoneToLast(TEXT("pinky_01_l"), FVector(5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_l"), FVector(3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_l"), FVector(1.8, 0, 0)); + AddBone(TEXT("thumb_01_l"), TEXT("hand_l"), FVector(2.0, -1.0, -2.6)); + AddBoneToLast(TEXT("thumb_02_l"), FVector(4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_l"), FVector(2.7, 0, 0)); + + AddBone(TEXT("hand_r"), TEXT("lowerarm_r"), FVector(-25.2, 0, 0)); + AddBone(TEXT("lowerarm_twist_01_r"), TEXT("lowerarm_r"), FVector(-14.0, 0, 0)); + AddBone(TEXT("lowerarm_twist_02_r"), TEXT("lowerarm_r"), FVector(-7.0, 0, 0)); + AddBone(TEXT("index_metacarpal_r"), TEXT("hand_r"), FVector(-3.5, -0.4, 2.1)); + AddBoneToLast(TEXT("index_01_r"), FVector(-5.9, 0.1, 0.3)); + AddBoneToLast(TEXT("index_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("index_03_r"), FVector(-2.4, 0, 0)); + AddBone(TEXT("middle_metacarpal_r"), TEXT("hand_r"), FVector(-3.3, -0.3, 0.1)); + AddBoneToLast(TEXT("middle_01_r"), FVector(-6.1, 0, 0.2)); + AddBoneToLast(TEXT("middle_02_r"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("middle_03_r"), FVector(-2.6, 0, 0)); + AddBone(TEXT("ring_metacarpal_r"), TEXT("hand_r"), FVector(-3.2, 0.2, -1.2)); + AddBoneToLast(TEXT("ring_01_r"), FVector(-6.0, 0.2, 0.4)); + AddBoneToLast(TEXT("ring_02_r"), FVector(-3.6, 0, 0)); + AddBoneToLast(TEXT("ring_03_r"), FVector(-2.5, 0, 0)); + AddBone(TEXT("pinky_metacarpal_r"), TEXT("hand_r"), FVector(-3.1, 0.7, -2.4)); + AddBoneToLast(TEXT("pinky_01_r"), FVector(-5.1, 0.1, 0.1)); + AddBoneToLast(TEXT("pinky_02_r"), FVector(-3.3, 0, 0)); + AddBoneToLast(TEXT("pinky_03_r"), FVector(-1.8, 0, 0)); + AddBone(TEXT("thumb_01_r"), TEXT("hand_r"), FVector(-2.0, 1.0, 2.6)); + AddBoneToLast(TEXT("thumb_02_r"), FVector(-4.4, 0, 0)); + AddBoneToLast(TEXT("thumb_03_r"), FVector(-2.7, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +void PoseAIRigDazUE::Configure() +{ + //daz has extra joints in the legs + rShinJoint = 5; + lShinJoint = 10; + lowerBodyNumOfJoints = 11; + + jointNames.Empty(); + boneVectors.Empty(); + parentIndices.Empty(); + + AddBone(TEXT("root"), TEXT("root"), FVector(0.0, 0, 0.0)); + AddBoneToLast(TEXT("hip"), FVector(0.0, -105.0, 0.0)); + AddBoneToLast(TEXT("pelvis"), FVector(0.0, -1.8, 0.0)); + + AddBone(TEXT("rThighBend"), TEXT("pelvis"), FVector(-7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("rThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("rShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("rFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("rToe"), FVector(0.0, 0, 14.0)); + + AddBone(TEXT("lThighBend"), TEXT("pelvis"), FVector(7.9, 10.6, -1.5)); + AddBoneToLast(TEXT("lThighTwist"), FVector(0.0, 21, 0)); + AddBoneToLast(TEXT("lShin"), FVector(0.0, 25.3, -1.2)); + AddBoneToLast(TEXT("lFoot"), FVector(0.0, 42.8, 1)); + AddBoneToLast(TEXT("lToe"), FVector(0.0, 0.0, 14.0)); + + AddBone(TEXT("abdomenLower"), TEXT("hip"), FVector(0.0, -1.7, -1.5)); + AddBoneToLast(TEXT("abdomenUpper"), FVector(0.0, -8.2, 1.2)); + AddBoneToLast(TEXT("chestLower"), FVector(0.0, -7.9, -0.4)); + AddBoneToLast(TEXT("chestUpper"), FVector(0.0, -13.1, -3.6)); + AddBoneToLast(TEXT("neckLower"), FVector(0.0, -18.3, -1.5)); + AddBoneToLast(TEXT("neckUpper"), FVector(0.0, -3.5, 1.5)); + AddBoneToLast(TEXT("head"), FVector(0.0, -4.9, -0.5)); + + AddBone(TEXT("lCollar"), TEXT("chestUpper"), FVector(3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("lShldrBend"), FVector(11.9, 1.7, 0)); + AddBoneToLast(TEXT("lShldrTwist"), FVector(11.6, 0, 0)); + AddBoneToLast(TEXT("lForearmBend"), FVector(14.4, -0.2, -0.5)); + + AddBone(TEXT("rCollar"), TEXT("chestUpper"), FVector(-3.5, -10.9, -1.6)); + AddBoneToLast(TEXT("rShldrBend"), FVector(-11.9, 1.7, 0)); + AddBoneToLast(TEXT("rShldrTwist"), FVector(-11.6, 0, 0)); + AddBoneToLast(TEXT("rForearmBend"), FVector(-14.4, -0.2, -0.5)); + + numBodyJoints = jointNames.Num(); + if (includeHands) { + AddBone(TEXT("lForearmTwist"), TEXT("lForearmBend"), FVector(12.1, 0, 0)); + AddBone(TEXT("lHand"), TEXT("lForearmTwist"), FVector(14.2, 0, -0.3)); + + AddBone(TEXT("lCarpal1"), TEXT("lHand"), FVector(0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("lIndex1"), FVector(7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("lIndex2"), FVector(3.9, 0, 0)); + AddBoneToLast(TEXT("lIndex3"), FVector(2.1, 0, 0)); + AddBone(TEXT("lCarpal2"), TEXT("lHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("lMid1"), FVector(7.5, -0.3, 0)); + AddBoneToLast(TEXT("lMid2"), FVector(4.3, 0, 0)); + AddBoneToLast(TEXT("lMid3"), FVector(2.5, 0, 0)); + AddBone(TEXT("lCarpal3"), TEXT("lHand"), FVector(0.8, -0.4, -0.8)); + AddBoneToLast(TEXT("lRing1"), FVector(6.9, -0.2, 0.0)); + AddBoneToLast(TEXT("lRing2"), FVector(4.0, 0, 0)); + AddBoneToLast(TEXT("lRing3"), FVector(2.2, 0, 0)); + AddBone(TEXT("lCarpal4"), TEXT("lHand"), FVector(0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("lPinky1"), FVector(6.5, 0.2, 0)); + AddBoneToLast(TEXT("lPinky2"), FVector(2.8, 0, 0)); + AddBoneToLast(TEXT("lPinky3"), FVector(1.7, 0, 0)); + AddBone(TEXT("lThumb1"), TEXT("lHand"), FVector(1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("lThumb2"), FVector(4.1, 0, 0)); + AddBoneToLast(TEXT("lThumb3"), FVector(3.0, 0, 0)); + + AddBone(TEXT("rForearmTwist"), TEXT("rForearmBend"), FVector(-12.1, 0, 0)); + AddBone(TEXT("rHand"), TEXT("rForearmTwist"), FVector(-14.2, 0, -0.3)); + + AddBone(TEXT("rCarpal1"), TEXT("rHand"), FVector(-0.4, -0.4, 1.1)); + AddBoneToLast(TEXT("rIndex1"), FVector(-7.6, -0.2, 0.1)); + AddBoneToLast(TEXT("rIndex2"), FVector(-3.9, 0, 0)); + AddBoneToLast(TEXT("rIndex3"), FVector(-2.1, 0, 0)); + AddBone(TEXT("rCarpal2"), TEXT("rHand"), FVector(0.7, -0.4, 0.2)); + AddBoneToLast(TEXT("rMid1"), FVector(-7.5, -0.3, 0)); + AddBoneToLast(TEXT("rMid2"), FVector(-4.3, 0, 0)); + AddBoneToLast(TEXT("rMid3"), FVector(-2.5, 0, 0)); + AddBone(TEXT("rCarpal3"), TEXT("rHand"), FVector(-0.8, -0.4, 0.8)); + AddBoneToLast(TEXT("rRing1"), FVector(-6.9, -0.2, 0)); + AddBoneToLast(TEXT("rRing2"), FVector(-4.0, 0, 0)); + AddBoneToLast(TEXT("rRing3"), FVector(-2.2, 0, 0)); + AddBone(TEXT("rCarpal4"), TEXT("rHand"), FVector(-0.7, -0.4, 1.7)); + AddBoneToLast(TEXT("rPinky1"), FVector(-6.5, 0.2, 0)); + AddBoneToLast(TEXT("rPinky2"), FVector(-2.8, 0, 0)); + AddBoneToLast(TEXT("rPinky3"), FVector(-1.7, 0, 0)); + AddBone(TEXT("rThumb1"), TEXT("rHand"), FVector(-1.4, 0.7, 1.6)); + AddBoneToLast(TEXT("rThumb2"), FVector(-4.1, 0, 0)); + AddBoneToLast(TEXT("rThumb3"), FVector(-3.0, 0, 0)); + } + numHandJoints = (jointNames.Num() - numBodyJoints) / 2; + rig = MakeStaticData(); +} + +#undef LOCTEXT_NAMESPACE + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp new file mode 100644 index 0000000..eec8f4a --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/PoseAIStructs.cpp @@ -0,0 +1,387 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAIStructs.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +// Utility conversion functions for compact representation and from arrays to vectors + +float UintB64ToUint(char a, char b) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + + return reverse_map[static_cast(a)] * 64 + reverse_map[static_cast(b)]; +} +uint32 UintB64ToUint(char a, char b, char c) { + static const float reverse_map[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 }; + return reverse_map[static_cast(a)] * 4096 + reverse_map[static_cast(b)] * 64 + reverse_map[static_cast(c)]; +} + +float FixedB64pairToFloat(char a, char b) { + static const float firstByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9384465070835368, 0.9697117733268197, 0.9384465070835368, 0.9384465070835368, 0.9697117733268197, 0.6257938446507083, 0.6570591108939912, 0.688324377137274, 0.7195896433805569, 0.7508549096238397, 0.7821201758671226, 0.8133854421104054, 0.8446507083536883, 0.8759159745969711, 0.907181240840254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, -0.9687347337567171, -0.9374694675134343, -0.9062042012701514, -0.8749389350268686, -0.8436736687835857, -0.8124084025403029, -0.78114313629702, -0.7498778700537372, -0.7186126038104543, -0.6873473375671715, -0.6560820713238886, -0.6248168050806058, -0.5935515388373229, -0.5622862725940401, -0.5310210063507572, -0.49975574010747437, -0.4684904738641915, -0.43722520762090866, -0.4059599413776258, -0.37469467513434296, -0.3434294088910601, -0.31216414264777725, -0.2808988764044944, -0.24963361016121155, -0.2183683439179287, 0.0, 0.0, 0.0, 0.0, 0.9697117733268197, 0.0, -0.18710307767464585, -0.155837811431363, -0.12457254518808014, -0.09330727894479729, -0.06204201270151444, -0.030776746458231585, 0.0004885197850512668, 0.03175378602833412, 0.06301905227161697, 0.09428431851489982, 0.12554958475818268, 0.15681485100146553, 0.18808011724474838, 0.21934538348803123, 0.2506106497313141, 0.28187591597459694, 0.3131411822178798, 0.34440644846116264, 0.3756717147044455, 0.40693698094772834, 0.4382022471910112, 0.46946751343429405, 0.5007327796775769, 0.5319980459208598, 0.5632633121641426, 0.5945285784074255 }; + static const float secondByte[256] = { 0.0f, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.030288226673180263, 0.030776746458231558, 0.030288226673180263, 0.030288226673180263, 0.030776746458231558, 0.025403028822667317, 0.025891548607718612, 0.026380068392769906, 0.0268685881778212, 0.027357107962872496, 0.02784562774792379, 0.028334147532975085, 0.02882266731802638, 0.029311187103077674, 0.02979970688812897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004885197850512946, 0.0009770395701025891, 0.0014655593551538837, 0.0019540791402051783, 0.002442598925256473, 0.0029311187103077674, 0.003419638495359062, 0.0039081582804103565, 0.004396678065461651, 0.004885197850512946, 0.00537371763556424, 0.005862237420615535, 0.006350757205666829, 0.006839276990718124, 0.0073277967757694185, 0.007816316560820713, 0.008304836345872008, 0.008793356130923302, 0.009281875915974597, 0.009770395701025891, 0.010258915486077186, 0.01074743527112848, 0.011235955056179775, 0.01172447484123107, 0.012212994626282364, 0.0, 0.0, 0.0, 0.0, 0.030776746458231558, 0.0, 0.012701514411333659, 0.013190034196384953, 0.013678553981436248, 0.014167073766487542, 0.014655593551538837, 0.015144113336590131, 0.015632633121641426, 0.01612115290669272, 0.016609672691744015, 0.01709819247679531, 0.017586712261846604, 0.0180752320468979, 0.018563751831949193, 0.019052271617000488, 0.019540791402051783, 0.020029311187103077, 0.02051783097215437, 0.021006350757205666, 0.02149487054225696, 0.021983390327308255, 0.02247191011235955, 0.022960429897410845, 0.02344894968246214, 0.023937469467513434, 0.024425989252564728, 0.024914509037616023 }; + return firstByte[static_cast(a)] + secondByte[static_cast(b)]; +} + +void FStringFixed12ToFloat(const FString& data, TArray& flatArray) { + flatArray.Reserve(flatArray.Num() + data.Len() / 2); + for (int i = 0; i + 1 < data.Len(); i += 2) + flatArray.Add(FixedB64pairToFloat(data[i], data[i + 1])); +} + +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray) { + quatArray.Reserve(quatArray.Num() + flatArray.Num() / 4); + for (int i = 0; i + 3 < flatArray.Num(); i += 4) + quatArray.Add(FQuat(flatArray[i], flatArray[i + 1], flatArray[i + 2], flatArray[i + 3])); +} + + +void ProcessFieldAsVector2D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector2D& fieldVector2D){ + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 2){ + fieldVector2D.X = (*value)[0]->AsNumber(); + fieldVector2D.Y = (*value)[1]->AsNumber(); + } +} + +void ProcessFieldAsVector3D(const TSharedPtr < FJsonObject > jsonObj, const FString& fieldName, FVector& fieldVector) { + const TArray < TSharedPtr < FJsonValue > >* value; + if (jsonObj->TryGetArrayField(fieldName, value) && value->Num() >= 3) { + fieldVector.X = (*value)[0]->AsNumber(); + fieldVector.Y = (*value)[1]->AsNumber(); + fieldVector.Z = (*value)[2]->AsNumber(); + } +} + + +void ProcessArrayAsVector2D(const TArray < float > value, FVector2D& fieldVector2D) { + if (value.Num() >= 2) { + fieldVector2D.X = value[0]; + fieldVector2D.Y = value[1]; + } +} + +void ProcessArrayAsVector3D(const TArray < float > value, FVector& fieldVector) { + if (value.Num() >= 3) { + fieldVector.X = value[0]; + fieldVector.Y = value[1]; + fieldVector.Z = value[2]; + } +} + +void SetAndCheckForChange(bool newValue, bool& field, bool& changeFlag) { + if (newValue != field) + changeFlag = true; + field = newValue; +} +void SetAndCheckForChange(float newValue, bool& field, bool& changeFlag) { + SetAndCheckForChange(newValue > 0.5f, field, changeFlag); +} +// End utility functions + + +bool FPoseAIHandshake::IncludesHands() const { + return !(mode == EPoseAiAppModes::RoomBodyOnly || mode == EPoseAiAppModes::PortraitBodyOnly); + +} +int32 FPoseAIHandshake::GetHandModelVersion() const { + return static_cast(handModelVersion) + 1; +} + +int32 FPoseAIHandshake::GetBodyModelVersion() const { + return static_cast(bodyModelVersion) + 2; +} + +FString FPoseAIHandshake::GetModeString() const { + switch (mode) { + case EPoseAiAppModes::Room: return "Room"; + case EPoseAiAppModes::Desktop: return "Desktop"; + case EPoseAiAppModes::Portrait: return "Portrait"; + case EPoseAiAppModes::RoomBodyOnly: return "RoomBodyOnly"; + case EPoseAiAppModes::PortraitBodyOnly: return "PortraitBodyOnly"; + default: + return "Room"; + } +} +FString FPoseAIHandshake::GetRigString() const { + switch (rig) { + case EPoseAiRigPresets::MetaHuman: return "MetaHuman"; + case EPoseAiRigPresets::UE4: return "UE4"; + case EPoseAiRigPresets::Mixamo: return "Mixamo"; + case EPoseAiRigPresets::MixamoAlt: return "MixamoAlt"; + case EPoseAiRigPresets::DazUE: return "DazUE"; + default: + return "MetaHuman"; + } +} + +FString FPoseAIHandshake::GetContextString() const { + return "Default"; +} + +FString FPoseAIHandshake::ToString() const { + return FString::Printf( + TEXT("{\"HANDSHAKE\":{" + "\"name\":\"Unreal LiveLink\"," + "\"rig\":\"%s\", " + "\"mode\":\"%s\", " + "\"face\":\"%s\", " + "\"context\":\"%s\", " + "\"whoami\":\"%s\", " + "\"signature\":\"%s\", " + "\"mirror\":\"%s\", " + "\"syncFPS\": %d, " + "\"cameraFPS\": %d, " + "\"modelVersion\": %d, " + "\"handModelVersion\": %d, " + "\"locomotion\":\"%s\", " + "\"packetFormat\": %d" + "}}"), + *(GetRigString()), + *(GetModeString()), + *(YesNoString(isFaceAnimating)), + *(GetContextString()), + *whoami, + *signature, + *(YesNoString(isMirrored)), + syncFPS, + cameraFPS, + GetBodyModelVersion(), + GetHandModelVersion(), + *(YesNoString(locomotionEvents)), + static_cast(packetFormat) + ); +} + + +bool FPoseAIHandshake::operator==(const FPoseAIHandshake& Other) const +{ + return rig == Other.rig && mode == Other.mode && syncFPS == Other.syncFPS && cameraFPS == Other.cameraFPS && isMirrored == Other.isMirrored && packetFormat == Other.packetFormat; +} + + + +FString FPoseAIModelConfig::ToString() const { + return FString::Printf( + TEXT("{\"CONFIG\":{" + "\"mirror\":\"%s\", " + "\"stepSensitivity\":%f, " + "\"armSensitivity\":%f, " + "\"crouchSensitivity\": %f, " + "\"jumpSensitivity\":%f" + "}}"), + *(YesNoString(isMirrored)), + stepSensitivity, + armSensitivity, + crouchSensitivity, + jumpSensitivity + ); +} + +bool FPoseAIEventPairBase::CheckTriggerAndUpdate() { + bool hasChanged = Count != InternalCount; + InternalCount = Count; + return hasChanged; +} + +void FPoseAIEventPair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Magnitude = FixedB64pairToFloat(compactString[3], compactString[4]); +} + +void FPoseAIGesturePair::ProcessCompact(const FString& compactString) { + Count = UintB64ToUint(compactString[0], compactString[1], compactString[2]); + Current = UintB64ToUint(compactString[3], compactString[4]); +} + +void FPoseAIEventStruct::ProcessCompactBody(const FString& compactString) { + TArray compactOrder = { &Footstep, &SidestepL, &SidestepR, &Jump, &FeetSplit, &ArmPump, &ArmFlex, &ArmGestureL, &ArmGestureR}; + if (compactString.Len() % 5 != 0) { + UE_LOG(LogTemp, Warning, TEXT("PoseAILiveLink: Invalid event string: %s."), *compactString); + return; + } + for (int i = 0; i < compactOrder.Num(); ++i) { + if (compactString.Len() < 5 * i + 5) + break; + compactOrder[i]->ProcessCompact(compactString.Mid(i * 5, 5)); + } +} + +void FPoseAIVisibilityFlags::ProcessCompact(const FString& visString) { + hasChanged = false; + SetAndCheckForChange(visString[0] != '0', isTorso, hasChanged); + SetAndCheckForChange(visString[1] != '0', isLeftLeg, hasChanged); + SetAndCheckForChange(visString[2] != '0', isRightLeg, hasChanged); + SetAndCheckForChange(visString[3] != '0', isLeftArm, hasChanged); + SetAndCheckForChange(visString[4] != '0', isRightArm, hasChanged); + if (visString.Len() > 5) + SetAndCheckForChange(visString[5] != '0', isFace, hasChanged); +} + +void FPoseAILiveValues::ProcessCompactScalarsBody(const FString& compactString) { + int32 idx = 0; + if(compactString.Len() < 14) return; + bodyHeight = FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) + 1.0f; + chestYaw = FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f; + stanceYaw = FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 180.0f; + stableFeet = UintB64ToUint(compactString[idx + 6], compactString[idx + 7]); + handZoneLeft = UintB64ToUint(compactString[idx + 8], compactString[idx + 9]); + handZoneRight = UintB64ToUint(compactString[idx + 10], compactString[idx + 11]); + isCrouching = UintB64ToUint(compactString[idx + 12], compactString[idx + 13]) > 0; +} + +void FPoseAILiveValues::ProcessCompactVectorsBody(const FString& compactString) { + //tbd - this could be simplified if we don't need to keep supported older versions of the api + int32 idx = 0; + if (compactString.Len() < 12) return; + upperBodyLean.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 180.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 180.0f + ); + idx += 4; + hipScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx+1]), + FixedB64pairToFloat(compactString[idx+2], compactString[idx+3]) + ); + idx += 4; + chestScreen.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]), + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) + ); + idx += 4; + if (compactString.Len() < idx + 12) return; + //ik vector rescaled by 0.25f to fit in fixed point range for compact format, so need to be rescaled by 4.0f + handIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + handIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + if (compactString.Len() < idx + 18) return; + rootTranslation.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkL.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; + footIkR.Set( + FixedB64pairToFloat(compactString[idx], compactString[idx + 1]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 2], compactString[idx + 3]) * 4.0f, + FixedB64pairToFloat(compactString[idx + 4], compactString[idx + 5]) * 4.0f + ); + idx += 6; +} + +void FPoseAILiveValues::ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbLeft.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessLeftHand = handObj->GetNumberField("Open"); +} + +void FPoseAILiveValues::ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject > handObj) { + FString Point = (handObj->HasTypedField("Point")) ? handObj->GetStringField("Point") : ""; + int32 idx = 0; + if (Point.Len() < idx + 4) return; + pointHandRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + idx += 4; + if (Point.Len() < idx + 4) return; + pointThumbRight.Set( + FixedB64pairToFloat(Point[idx], Point[idx + 1]), + FixedB64pairToFloat(Point[idx + 2], Point[idx + 3]) + ); + if (handObj->HasTypedField("Open")) + opennessRightHand = handObj->GetNumberField("Open"); +} + + +void FPoseAIVisibilityFlags::ProcessVerbose(FPoseAIScalarStruct& scalars) { + hasChanged = false; + SetAndCheckForChange(scalars.VisTorso, isTorso, hasChanged); + SetAndCheckForChange(scalars.VisLegL, isLeftLeg, hasChanged); + SetAndCheckForChange(scalars.VisLegR, isRightLeg, hasChanged); + SetAndCheckForChange(scalars.VisArmL, isLeftArm, hasChanged); + SetAndCheckForChange(scalars.VisArmR, isRightArm, hasChanged); +} + + +const FString FPoseAILiveValues::fieldPointScreen = FString(TEXT("PointScreen")); +const FString FPoseAILiveValues::fieldThumbScreen = FString(TEXT("ThumbScreen")); + +void FPoseAIScalarStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody) { + FJsonObjectConverter::JsonObjectToUStruct(scaBody.ToSharedRef(), this); +} + +void FPoseAIEventStruct::ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody) { + FJsonObjectConverter::JsonObjectToUStruct(eveBody.ToSharedRef(), this); +} + +void FPoseAIVerbose::ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj) { + FJsonObjectConverter::JsonObjectToUStruct(jsonObj.ToSharedRef(), this); +} + + +void FPoseAILiveValues::ProcessVerboseBody(const FPoseAIVerbose& verbose){ + bodyHeight = verbose.Scalars.BodyHeight; + stableFeet = FMath::RoundToInt((float)verbose.Scalars.StableFoot); + stanceYaw = verbose.Scalars.StanceYaw * 180.0f; + chestYaw = verbose.Scalars.ChestYaw * 180.0f; + isCrouching = verbose.Scalars.IsCrouching > 0.5f; + handZoneLeft = verbose.Scalars.HandZoneL; + handZoneRight= verbose.Scalars.HandZoneR; + ProcessArrayAsVector2D(verbose.Vectors.HipLean, upperBodyLean); + upperBodyLean *= 180.0f; + ProcessArrayAsVector2D(verbose.Vectors.HipScreen, hipScreen); + ProcessArrayAsVector2D(verbose.Vectors.ChestScreen, chestScreen); + ProcessArrayAsVector3D(verbose.Vectors.HandIkL, handIkL); + ProcessArrayAsVector3D(verbose.Vectors.HandIkR, handIkR); + ProcessArrayAsVector3D(verbose.Vectors.Hip, rootTranslation); + ProcessArrayAsVector3D(verbose.Vectors.FootIkL, footIkL); + ProcessArrayAsVector3D(verbose.Vectors.FootIkR, footIkR); +} + + + +void FPoseAILiveValues::ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandLeft); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbLeft); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkL); +} + +void FPoseAILiveValues::ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand){ + if (vecHand==nullptr) + return; + ProcessFieldAsVector2D(vecHand, fieldPointScreen, pointHandRight); + ProcessFieldAsVector2D(vecHand, fieldThumbScreen, pointThumbRight); + ProcessFieldAsVector3D(vecHand, "FingerIk", fingerIkR); +} + + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp new file mode 100644 index 0000000..70554f9 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Private/SPoseAILiveLinkWidget.cpp @@ -0,0 +1,291 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "SPoseAILiveLinkWidget.h" +#include "PoseAILiveLinkSourceFactory.h" +#include "PoseAILiveLinkNetworkSource.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +TWeakPtr SPoseAILiveLinkWidget::source = nullptr; + +// right now this is manually aligned with the enums but should be a lookup to keep it from breaking +static TArray PoseAI_Modes = { "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" }; +static TArray PoseAI_Rigs = { "MetaHuman", "UE4", "Mixamo", "DazUE"}; + +const FString SPoseAILiveLinkWidget::section = "PoseLiveLink.SourceConfig"; +int32 SPoseAILiveLinkWidget::portNum = PoseAILiveLinkNetworkSource::portDefault; +int32 SPoseAILiveLinkWidget::syncFPS = 60; +int32 SPoseAILiveLinkWidget::cameraFPS = 60; +int32 SPoseAILiveLinkWidget::modeIndex = 0; +int32 SPoseAILiveLinkWidget::rigIndex = 0; +bool SPoseAILiveLinkWidget::isMirrored = false; +bool SPoseAILiveLinkWidget::isIPv6 = false; + +BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION +void SPoseAILiveLinkWidget::Construct(const FArguments& InArgs) +{ + GConfig->GetBool(*section, TEXT("isIPv6"), isIPv6, GEditorIni); + GConfig->GetBool(*section, TEXT("isMirrored"), isMirrored, GEditorIni); + + GConfig->GetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + if (modeIndex < 0 || modeIndex >= PoseAI_Modes.Num()) + modeIndex = 0; + GConfig->GetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + if (rigIndex < 0 || rigIndex >= PoseAI_Rigs.Num()) + rigIndex = 0; + + GConfig->GetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + + ChildSlot + [ + SNew(SBox).WidthOverride(300) + [ + SNew(SVerticalBox) + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("UseIPv6", "Connect via an IPv6 socket")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(ipv6CheckBox, SCheckBox) + .IsChecked(isIPv6 ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.75f) + [ + SNew(STextBlock).Text(LOCTEXT("PortIPv4", "Port for IPv4 (0 if unused)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).FillWidth(0.25f) + [ + SAssignNew(portInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdatePort) + .Text(FText::FromString(FString::FromInt(portNum))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.65f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleMode", "Toggle Camera Mode")) + + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.45f) + + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleModeClicked) + [ + SAssignNew(modeInput, STextBlock) + .Text(FText::FromString(PoseAI_Modes[modeIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.6f) + [ + SNew(STextBlock) + .Text(LOCTEXT("ToggleRig", "Toggle rig format")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.4f) + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnToggleRigClicked) + [ + SAssignNew(rigInput, STextBlock) + .Text(FText::FromString(PoseAI_Rigs[rigIndex])) + ] + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock) + .Text(LOCTEXT("IsMirrored", "Mirror camera (flip left/right)")) + ] + + SHorizontalBox::Slot().VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(mirroredCheckBox, SCheckBox) + .IsChecked(isMirrored ? ECheckBoxState::Checked : ECheckBoxState::Unchecked) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("SyncFPS", "Smoothed FPS (by app)")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(syncFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateSyncFPS) + .Text(FText::FromString(FString::FromInt(syncFPS))) + ] + ] + + SVerticalBox::Slot().AutoHeight() + [ + SNew(SHorizontalBox) + + SHorizontalBox::Slot().Padding(1, 3, 3, 3).VAlign(VAlign_Center).HAlign(HAlign_Left).FillWidth(0.85f) + [ + SNew(STextBlock).Text(LOCTEXT("CameraFPS", "Request Camera FPS")) + ] + + SHorizontalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Center).FillWidth(0.15f) + [ + SAssignNew(cameraFpsInput, SEditableTextBox) + .OnTextCommitted(this, &SPoseAILiveLinkWidget::UpdateCameraFPS) + .Text(FText::FromString(FString::FromInt(cameraFPS))) + ] + ] + + SVerticalBox::Slot().Padding(3, 3, 1, 3).VAlign(VAlign_Center).HAlign(HAlign_Right).AutoHeight() + [ + SNew(SButton) + .OnClicked(this, &SPoseAILiveLinkWidget::OnOkClicked) + [ + SNew(STextBlock) + .Text(LOCTEXT("OK", "OK")) + ] + ] + ] + ]; +} +END_SLATE_FUNCTION_BUILD_OPTIMIZATION + + +void SPoseAILiveLinkWidget::UpdatePort(const FText& InText, ETextCommit::Type type) +{ + portNum = FCString::Atoi(*(InText.ToString())); +} + + +bool SPoseAILiveLinkWidget::IsPortValid() const +{ + if (portNum < 1028 || portNum > 49151) { + FLiveLinkLog::Warning(TEXT("PoseAI: %d is an invalid port number. Set a valid port number (>1028 and <49151)."), portNum); + return false; + } + + if (!PoseAILiveLinkNetworkSource::IsValidPort(portNum)) { + FLiveLinkLog::Warning(TEXT("PoseAI: Cannot set two sources with the same port. %d is in use already."), portNum); + return false; + } + return true; +} + + +void SPoseAILiveLinkWidget::UpdateSyncFPS(const FText& InText, ETextCommit::Type type) +{ + syncFPS = FCString::Atoi(*(InText.ToString())); + if (syncFPS < 0) { + syncFPS = 0; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + +void SPoseAILiveLinkWidget::UpdateCameraFPS(const FText& InText, ETextCommit::Type type) +{ + cameraFPS = FCString::Atoi(*(InText.ToString())); + if (cameraFPS < 24) { + cameraFPS = 24; + } + if (syncFPS < cameraFPS && syncFPS > 0) + syncFPS = cameraFPS; + + GConfig->SetInt(*section, TEXT("cameraFPS"), cameraFPS, GEditorIni); + GConfig->SetInt(*section, TEXT("syncFPS"), syncFPS, GEditorIni); + GConfig->Flush(false, GEditorIni); +} + + +void SPoseAILiveLinkWidget::disableExistingSource() +{ + TSharedPtr linkSource = source.Pin(); + if (linkSource.IsValid()) { + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: disabling existing source")); + ((PoseAILiveLinkNetworkSource*)linkSource.Get())->disable(); + } +} +FPoseAIHandshake SPoseAILiveLinkWidget::GetHandshake() +{ + FPoseAIHandshake handshake = FPoseAIHandshake(); + handshake.isMirrored = isMirrored; + handshake.rig = static_cast(rigIndex); + handshake.mode = static_cast(modeIndex); + handshake.syncFPS = syncFPS; + handshake.cameraFPS = cameraFPS; + UE_LOG(LogTemp, Display, TEXT("PoseAI LiveLink: Set handshake to %s"), *(handshake.ToString())); + return handshake; +} + +TSharedPtr SPoseAILiveLinkWidget::CreateSource(const FString& connectionString) +{ + return PoseAILiveLinkNetworkSource::MakeSource(GetHandshake(), portNum, isIPv6); +} + +FReply SPoseAILiveLinkWidget::OnToggleModeClicked() +{ + modeIndex = (modeIndex+1) % PoseAI_Modes.Num(); + modeInput->SetText(FText::FromString(PoseAI_Modes[modeIndex])); + GConfig->SetInt(*section, TEXT("CameraMode"), modeIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnToggleRigClicked() +{ + rigIndex = (rigIndex + 1) % PoseAI_Rigs.Num(); + rigInput->SetText(FText::FromString(PoseAI_Rigs[rigIndex])); + GConfig->SetInt(*section, TEXT("Rig"), rigIndex, GEditorIni); + GConfig->Flush(false, GEditorIni); + return FReply::Handled(); +} + + +FReply SPoseAILiveLinkWidget::OnOkClicked() +{ + ReadCheckBox(mirroredCheckBox, isMirrored); + ReadCheckBox(ipv6CheckBox, isIPv6); + + GConfig->SetBool(*section, TEXT("isMirror"), isMirrored, GEditorIni); + + if (IsPortValid()) { + GConfig->SetInt(*section, TEXT("PortNumber"), portNum, GEditorIni); + GConfig->Flush(false, GEditorIni); + FString connectionString = ""; + TSharedPtr src = CreateSource(connectionString); + callback.Execute(src, connectionString); + FLiveLinkLog::Info( + TEXT("PoseAI: Setup source. Rig is in %s format, %s and %s."), + *PoseAI_Rigs[rigIndex], + *FString(isMirrored ? TEXT("mirrored to camera"): TEXT("third person")) + ); + } + return FReply::Handled(); +} + +void SPoseAILiveLinkWidget::ReadCheckBox(TWeakPtr& checkBox, bool& readTo) +{ + TSharedPtr pin = checkBox.Pin(); + if (pin) + readTo = (pin->GetCheckedState() == ECheckBoxState::Checked); +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..a42cb5c --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIGroundPenetration.h @@ -0,0 +1,61 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.generated.h" + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIGroundPenetration : public FAnimNode_SkeletalControlBase +{ + GENERATED_USTRUCT_BODY() + + /** Name of bone to apply live movement to, usually either root or pelvis/hip. **/ + UPROPERTY(EditAnywhere, Category = SkeletalControl) + FBoneReference BoneToModify; + + /** Set to true to always have contact with ground **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration, meta = (PinShownByDefault)) + bool PinToFloor = false; + + /** These bones will be checked for ground penetration **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray BonesToCheck; + + /** These sockets will be checked for ground pnetration. **/ + UPROPERTY(EditAnywhere, Category = GroundPenetration) + TArray SocketsToCheck; + + + + + TArray SocketsBoneReference; + TArray SocketsLocalTransform; + + FAnimNode_PoseAIGroundPenetration(); + + // FAnimNode_Base interface + virtual void GatherDebugData(FNodeDebugData& DebugData) override; + // End of FAnimNode_Base interface + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; + + + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h new file mode 100644 index 0000000..c65676f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/AnimNode_PoseAIHandTarget.h @@ -0,0 +1,64 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "BoneContainer.h" +#include "BonePose.h" +#include "BoneControllers/AnimNode_SkeletalControlBase.h" +#include "BoneControllers/AnimNode_TwoBoneIK.h" + +#include "AnimNode_PoseAIHandTarget.generated.h" + + +/** + * Debugging node that displays the current value of a bone in a specific space. + */ +USTRUCT() +struct POSEAILIVELINK_API FAnimNode_PoseAIHandTarget : public FAnimNode_TwoBoneIK +{ + GENERATED_USTRUCT_BODY() + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference SpineFirst; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference LeftUpperArm; + + /** Name of bone to control. This is the main bone chain to modify from. **/ + UPROPERTY(EditAnywhere, Category = IK) + FBoneReference RightUpperArm; + + // for now we hide this feature as it can create unwelcome jumps in wrist position + /** If specified, will use index finger tip for solution. **/ + UPROPERTY() + FBoneReference UseIndexFingerTip; + + + /** Special IK control info from PoseAI movement component. This is NOT a location vector. **/ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = Effector, meta = (PinShownByDefault)) + FVector PoseAiIkVector = FVector::ZeroVector; + + FCompactPoseBoneIndex IKBoneCompactPoseIndex; + FCompactPoseBoneIndex SpineFirstIndex; + FCompactPoseBoneIndex LeftUpperArmIndex; + FCompactPoseBoneIndex RightUpperArmIndex; + FCompactPoseBoneIndex IndexFingerTipIndex; +public: + FAnimNode_PoseAIHandTarget(); + + + + // FAnimNode_SkeletalControlBase interface + virtual void EvaluateSkeletalControl_AnyThread(FComponentSpacePoseContext& Output, TArray& OutBoneTransforms) override; + virtual bool IsValidToEvaluate(const USkeleton* Skeleton, const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +private: + // FAnimNode_SkeletalControlBase interface + virtual void InitializeBoneReferences(const FBoneContainer& RequiredBones) override; + // End of FAnimNode_SkeletalControlBase interface + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h new file mode 100644 index 0000000..977dea1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEndpoint.h @@ -0,0 +1,128 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "IPAddress.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "SocketSubsystem.h" + + +TSharedPtr BuildUdpSocket(FString& description, FName protocolType, int32 port); + + +/** + * Implements a more generic endpoint to allow for both IPv6 and IPv4 networks, based on Epic Games IPv4 endpoint from the Networking module. + * + * Mainly is a wrapper around an FInternetAddr with the helper functions needed to use the networking code with only minor modifications. + * + */ +struct FPoseAIEndpoint +{ + /** Holds the endpoint's IP address. */ + TSharedPtr Address; + + /** Holds the endpoint's port number. */ + uint16 Port; + +public: + + /** Default constructor. */ + FPoseAIEndpoint() { } + + + /** + * Creates and initializes a new endpoint from a given FInternetAddr object. + * + * Note: this constructor will be removed after the socket subsystem has been refactored. + * + * @param InternetAddr The Internet address. + */ + + + FPoseAIEndpoint(const TSharedPtr& InternetAddr) + { + check(InternetAddr.IsValid()); + + int32 OutPort; + Address = InternetAddr; + InternetAddr->GetPort(OutPort); + Port = OutPort; + } + + bool IsValid() const { return Address != nullptr && Address.IsValid(); } + +public: + + /** + * Compares this endpoint with the given endpoint for equality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are equal, false otherwise. + */ + bool operator==(const FPoseAIEndpoint& Other) const + { + return ((Address == Other.Address)); + } + + /** + * Compares this address with the given endpoint for inequality. + * + * @param Other The endpoint to compare with. + * @return true if the endpoints are not equal, false otherwise. + */ + bool operator!=(const FPoseAIEndpoint& Other) const + { + return (Address != Other.Address); + } + + +public: + + + /** + * Gets a string representation for this endpoint. + * + * @return String representation. + * @see Parse, ToText + */ + POSEAILIVELINK_API FString ToString() const; + + /** + * Gets the display text representation. + * + * @return Text representation. + * @see ToString + */ + FText ToText() const + { + return FText::FromString(ToString()); + } + + TSharedRef ToInternetAddr() const + { + if (CachedSocketSubsystem == nullptr) + Initialize(); + + check(CachedSocketSubsystem != nullptr && "Networking module not loaded and initialized"); + TSharedRef InternetAddr = CachedSocketSubsystem->CreateInternetAddr(Address->GetProtocolType()); + { + InternetAddr->SetRawIp(Address->GetRawIp()); + InternetAddr->SetPort(Address->GetPort()); + } + + return InternetAddr; + } + +public: + + /** Initializes the IP endpoint functionality. */ + static void Initialize(); + + + +private: + /** ISocketSubsystem::Get() is not thread-safe, so we cache it here. */ + static ISocketSubsystem* CachedSocketSubsystem; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h new file mode 100644 index 0000000..3eecfa0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIEventDispatcher.h @@ -0,0 +1,395 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "Async/Async.h" +#include "LiveLinkTypes.h" +#include "PoseAIStructs.h" +#include "PoseAIEventDispatcher.generated.h" + + +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIDisconnect, const FLiveLinkSubjectName&); +DECLARE_MULTICAST_DELEGATE_OneParam(FPoseAIHandshakeUpdate, const FPoseAIHandshake&); +DECLARE_MULTICAST_DELEGATE_TwoParams(FPoseAIConfigUpdate, const FLiveLinkSubjectName&, FPoseAIModelConfig); + +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAISubjectConnected, const FLiveLinkSubjectName&, SubjectName, bool, isReconnection); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIRegisteredAs, const FLiveLinkSubjectName&, SubjectName, FName, ConnectionName); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIFrameReceived); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIVisibilityChange, const FPoseAIVisibilityFlags&, Flags); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAILiveValuesUpdate, const FPoseAILiveValues&, LiveValues); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootstepEvent, float, height, bool, isLeftFoot); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAISidestepEvent, bool, isLeftStep); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIFootsplitEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmpumpEvent, float, height); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FPoseAIArmflexEvent, float, width, bool, isExpanding); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmjackEvent,bool, isRising); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIArmflapEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIJumpEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAICrouchEvent, bool, isCrouching); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIHandToZoneEvent, int32, zone); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FPoseAIArmGestureEvent, int32, armGesture); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIStationaryEvent); +DECLARE_DYNAMIC_MULTICAST_DELEGATE(FPoseAIResetLivePositionEvent); + + +UCLASS(ClassGroup = (PoseAI)) +class POSEAILIVELINK_API UStepCounter : public UObject +{ + GENERATED_BODY() + +public: + /** Convenience setter for timeout and fadeduration at the same time*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + UStepCounter* SetProperties(float timeoutIn = 0.5f, float fadeDuration = 0.2f); + + /** clears all steps. Optionally fades speed over */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Steptracker") + void Halt(bool fade); + + /** Average step distance in [body width, body height] units per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float DistancePerSecond(); + + /** Average number of steps per second. If most recent step was longer than ago, speed is faded to zero */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float StepsPerSecond(); + + /** Time since last step in seconds*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float TimeSinceLastStep(); + + /** most recent step distance*/ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Steptracker") + float LastDistance(); + + + /** time since last step when motion begins to slow*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float timeout = 0.5f; + + /** after a next step timeout, the speed is faded to zero over this duration*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Steptracker") + float fadeDurationOnTimeout = 0.2f; + + /** total steps registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + int32 totalSteps = 0; + + /** total distance registered by the stepcounter*/ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Steptracker") + float totalDistance = 0.0f; + + void RegisterStep(float stepDistance); + + UStepCounter(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) + { + + times_.SetNumUninitialized(num_to_track); + heights_.SetNumUninitialized(num_to_track); + } + + +private: + float CheckIfActiveAndFade(); + const int32 num_to_track = 4; + int32 num_ = 0; + int32 tail_ = -1; + + float steps_per_second_ = 0.0f; + float distance_per_second_ = 0.0f; + FDateTime last_time_ = FDateTime::Now(); + TArray times_; + TArray heights_; +}; + + +class UPoseAIEventDispatcher; + +UCLASS(ClassGroup = (PoseAI), meta = (BlueprintSpawnableComponent)) +class POSEAILIVELINK_API UPoseAIMovementComponent : public UActorComponent +{ + GENERATED_BODY() + friend UPoseAIEventDispatcher; + + public: + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple portss (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, FString& myIP, int32 portNum=8080, bool isIPv6 = false); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP); + + /** Sends a message to the connected PoseCamera to reconfigure the model with user settings */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void ChangeModelConfig(FPoseAIModelConfig config); + + /** sends disconnect message to app and closes source, freeing up Port*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(); + + /** sends disconnect message to app but does not clsoe source */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Disconnect(); + + /** Get the LiveLink subject name associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectName() { return subjectName; } + + /** Get the LiveLink subject name for facial animation associated with this component */ + UFUNCTION(BlueprintCallable, BlueprintPure, Category = "PoseAI Setup") + FLiveLinkSubjectName GetSubjectFaceName(); + + /** Will assign component to the next available Pose AI LiveLink source. Useful if sources managed with a preswet (Otherwise prefer use of AddSource nodes) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void RegisterAsFirstAvailable(); + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + static void SetHandshake(const FPoseAIHandshake& handshake); + + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool RegisterAs(FLiveLinkSubjectName name, bool siezeIfTaken = true); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void Deregister(); + + /** Sets rig height, which scales root motion, and allows scaling per dimension (i.e. set Y=0 for no motion to/from camera) */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ScaleMotion(float RigHeight=170.0f, FVector Scale=FVector(1.0f, 1.0f,1.0f)); + + /** compensates for an active source's camera rotation in the real world */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void SetLiveCameraRotation(float pitch, float yaw = 0.0f, float roll=0.0f); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseAsBaseTranslation(); + + /** Have player stand up straight and face screen as part of configuration */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void UseCurrentPoseToOrientCamera(); + + /** Remove all live root motion (sets scalemotion to zero)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Configuration") + void ZeroMotion(); + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FDateTime lastFrameReceived; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAIVisibilityFlags visibilityFlags; + + UPROPERTY(BlueprintReadOnly, VisibleAnywhere, Category = "PoseAI Events") + FPoseAILiveValues mostRecentValues; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* footsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* leftsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* rightsteps; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* feetsplits; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armpumps ; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflexes; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armjacks; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapL; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* armflapR; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Events") + UStepCounter* jumps; + + /********** events ********************/ + + /** when the component succeesfully registered with a connection (i.e. a pose camera joined) */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIRegisteredAs onRegistered; + + /** when any body part visisbility flag changes */ + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIVisibilityChange onVisibilityChange; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAILiveValuesUpdate onLiveValues; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootstepEvent onFootstep; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIFootsplitEvent onFeetsplit; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepLeftFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISidestepEvent onSidestepRightFoot; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmpumpEvent onArmpump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflexEvent onArmflex; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmflapEvent onArmflapL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmjackEvent onArmjack; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureLeft; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIArmGestureEvent onArmGestureRight; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIJumpEvent onJump; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAICrouchEvent onCrouch; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneL; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIHandToZoneEvent onHandToZoneR; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIStationaryEvent onStationary; + + void SetLiveValues(FPoseAILiveValues values) { + mostRecentValues = values; + } + + virtual void InitializeComponent() override { + Super::InitializeComponent(); + InitializeObjects(); + } + +private: + FLiveLinkSubjectName subjectName; + FLiveLinkSubjectName subjectFaceName; + + void InitializeObjects(); + +}; + + +/** + * Dispatcher to transmit events to blueprints and c++, can access events from all sources + */ +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAIEventDispatcher : public UObject +{ +GENERATED_BODY() +public: + /** Gets the singleton dispatcher. Use this for binding or to get named components */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + static UPoseAIEventDispatcher* GetDispatcher() { + if (theInstance==nullptr) { + theInstance = NewObject(); + theInstance->AddToRoot(); + UE_LOG(LogTemp, Display, TEXT("PoseAILiveLink: Creating EventDispatcher.")); + } + return theInstance; + } + + /** sets the handshake for all sources */ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void SetHandshake(const FPoseAIHandshake& handshake); + + FPoseAIHandshakeUpdate handshakeUpdate; + FPoseAIConfigUpdate modelConfigUpdate; + FPoseAIDisconnect disconnect; + FPoseAIDisconnect closeSource; + + /** Adds a LiveLink source listening for Posecam at the designated port, but will overwrite an existing listener so developer needs to manage if using multiple ports (or use the AddSourceNextOpenPort node instead)*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSource(const FPoseAIHandshake& handshake, bool isIPv6, int32 portNum, FString& myIP, FLiveLinkSubjectName& subject); + + /** Adds a LiveLink source listening for Posecam at the next open port beginning at 8080*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + bool AddSourceNextOpenPort(const FPoseAIHandshake& handshake, bool isIPv6, int32& portNum, FString& myIP, FLiveLinkSubjectName& subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Setup") + void CloseSource(FLiveLinkSubjectName subject); + + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + FLiveLinkSubjectName GetFirstUnboundSubject(bool excludeIdleSubjects = true); + + /** Convenience event to tell animation blueprin*/ + UFUNCTION(BlueprintCallable, Category = "PoseAI Events") + void BroadcastResetLivePosition(); + + /** convenience accessor for animation blueprints in one source projects, but no guarantee reference is valid or preserve. Use a proper link between ABP and BP to refer in a more stable manner */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI Setup") + UPoseAIMovementComponent* LastMovementComponent; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAISubjectConnected subjectConnected; + + UPROPERTY(BlueprintAssignable, Category = "PoseAI Events") + FPoseAIResetLivePositionEvent resetLivePositionEvent; + + // Connection driven events + void BroadcastCloseSource(const FLiveLinkSubjectName& subjectName); + void BroadcastConfigUpdate(const FLiveLinkSubjectName& subjectName, FPoseAIModelConfig config); + void BroadcastDisconnect(const FLiveLinkSubjectName& subjectName); + void BroadcastFrameReceived(const FLiveLinkSubjectName& subjectName); + void BroadcastSubjectConnected(const FLiveLinkSubjectName& subjectName); + + // Pose Camera driven events + void BroadcastArmpumps(const FLiveLinkSubjectName& subjectName, float stepHeight); + void BroadcastArmflexes(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastArmjacks(const FLiveLinkSubjectName& subjectName, bool isRising); + void BroadcastArmGestureL(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastArmGestureR(const FLiveLinkSubjectName& subjectName, int32 gesture); + void BroadcastCrouches(const FLiveLinkSubjectName& subjectName, bool isCrouching); + void BroadcastFootsteps(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isLeftStep); + void BroadcastFeetsplits(const FLiveLinkSubjectName& subjectName, float stepHeight, bool isExpanding); + void BroadcastHandToZoneL(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastHandToZoneR(const FLiveLinkSubjectName& subjectName, int32 zone); + void BroadcastJumps(const FLiveLinkSubjectName& subjectName); + void BroadcastLiveValues(const FLiveLinkSubjectName& subjectName, FPoseAILiveValues values); + void BroadcastSidestepL(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastSidestepR(const FLiveLinkSubjectName& subjectName, bool isLeftStep); + void BroadcastStationary(const FLiveLinkSubjectName& subjectName); + void BroadcastVisibilityChange(const FLiveLinkSubjectName& subjectName, FPoseAIVisibilityFlags visibilityFlags); + + bool RegisterComponentByName(UPoseAIMovementComponent* component, const FLiveLinkSubjectName& name, bool siezeIfTaken); + void RegisterComponentForFirstAvailableSubject(UPoseAIMovementComponent* component); + + bool HasComponent(const FLiveLinkSubjectName& name, UPoseAIMovementComponent*& component); + + void BroadcastResetZeroLivePosition(); + +private: + static UPoseAIEventDispatcher* theInstance; + const double timeoutInSeconds = 60.0; + TQueue componentQueue; + UPROPERTY() + TMap componentsByName; + TMap knownConnectionsWithTime; + UPoseAIEventDispatcher() : UObject() {}; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h new file mode 100644 index 0000000..75d849b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLink.h @@ -0,0 +1,20 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h new file mode 100644 index 0000000..13d5dee --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkFaceSubSource.h @@ -0,0 +1,105 @@ +// Copyright Pose AI Ltd 2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "Json.h" + + +/** + *A child object for the LiveLink sources to manage the face animation as a supplementary livelink subject + */ +class POSEAILIVELINK_API PoseAILiveLinkFaceSubSource +{ + +public: + /* Prefer using the AddSource method for setup */ + PoseAILiveLinkFaceSubSource(FLiveLinkSubjectKey& poseSubjectKey, ILiveLinkClient* liveLinkClient); + bool AddSubject(FCriticalSection& InSynchObject); + bool RequestSubSourceShutdown(); + void UpdateFace(TSharedPtr jsonPose); + +private: + + FLiveLinkSubjectKey subjectKey; + FName subjectName = "FacePoseAI"; // will be overwritten on initialization + ILiveLinkClient* liveLinkClient = nullptr; + FLiveLinkSkeletonStaticData StaticData; +}; + + +UENUM(BlueprintType, Category = "PoseAI Animation", meta = (Experimental)) +enum class PoseAIFaceBlendShape : uint8 +{ + // Left eye blend shapes + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + // Right eye blend shapes + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + // Jaw blend shapes + JawForward, + JawLeft, + JawRight, + JawOpen, + // Mouth blend shapes + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + // Brow blend shapes + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + // Cheek blend shapes + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + // Nose blend shapes + NoseSneerLeft, + NoseSneerRight, + TongueOut, + MAX +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h new file mode 100644 index 0000000..56ac13d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNativeSource.h @@ -0,0 +1,72 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + +/** + * Source from in game engine framework. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNativeSource : public ILiveLinkSource +{ + + /* + Use a static method to add source via a sole shared ptr with only ownership by LiveLinkClient, and pass back weak pointer to caller. + LiveLink really seems to want to own the only shared pointer or cleanup can crash. + */ +public: + static TWeakPtr AddSource(FName subjectName, const FPoseAIHandshake& handshake); + bool AddSubject(); + void ReceivePacket(const FString& recvMessage); + + PoseAILiveLinkNativeSource(FName subjectName, const FPoseAIHandshake& handshake); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {}; + +public: + TSharedPtr rig; + void disable(); + void UpdatePose(TSharedPtr jsonPose); + +private: + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + FName subjectName = "PoseAILocalCam"; + ILiveLinkClient* liveLinkClient = nullptr; + FCriticalSection InSynchObject; + FPoseAIHandshake handshake; + TUniquePtr faceSubSource; + + mutable FText status; + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h new file mode 100644 index 0000000..6fd81a0 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkNetworkSource.h @@ -0,0 +1,144 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "ILiveLinkSource.h" +#include "ILiveLinkClient.h" +#include "LiveLinkClient.h" +#include "LiveLinkSourceSettings.h" +#include "LiveLinkSubjectSettings.h" +#include "LiveLinkFrameInterpolationProcessor.h" +#include "LiveLinkFramePreProcessor.h" +#include "LiveLinkFrameTranslator.h" +#include "Roles/LiveLinkAnimationRole.h" +#include "LiveLinkTypes.h" +#include "LiveLinkLog.h" +#include "HAL/RunnableThread.h" +#include "Json.h" +#include "PoseAIRig.h" +#include "PoseAILiveLinkServer.h" +#include "PoseAIStructs.h" +#include "PoseAILiveLinkFaceSubSource.h" + + + +struct POSEAILIVELINK_API PoseAIPortRecord { + FGuid source; + FName connectionName; + FLiveLinkSubjectKey subjectKey; +}; + +class PoseAILiveLinkSingleSourceListener; + + +/** + * Redesigned so that each phone is associated with a single source, on a single port, for simplicity. + * Each source maintains its own "server" object, which generates the UDP socket, a listener and a sender class on their own threads. + * The server feeds into the EventDispatcher system to trigger connection events. Incoming packets are processed by the Rig class to + * trigger frame events and update the LiveLink pose source information. + */ +class POSEAILIVELINK_API PoseAILiveLinkNetworkSource : public ILiveLinkSource +{ +public: + + /* + * method to add a source from code, instead of relying on presets.Exposed to blueprints via the PoseAI movement component. + * returns true if source was added and fills in subjectNamd with the name of the source's sole subject in the LiveLink system + */ + static bool AddSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6, FLiveLinkSubjectName& subjectName); + static TSharedPtr MakeSource(const FPoseAIHandshake& handshake, int32 portNum, bool isIPv6); + + /* Prefer the MakeSource factory method to setup source correctly */ + PoseAILiveLinkNetworkSource(const FPoseAIHandshake& handshake, int32 port, bool useIPv6); + + // standard Live Link source methods + virtual bool CanBeDisplayedInUI() const { return true; } + virtual TSubclassOf< ULiveLinkSourceSettings > GetSettingsClass() const override { return nullptr; } + virtual FText GetSourceType() const; + virtual FText GetSourceMachineName() const; + virtual FText GetSourceStatus() const { return status; } + virtual void InitializeSettings(ULiveLinkSourceSettings* Settings) override; + virtual bool IsSourceStillValid() const override; + virtual void OnSettingsChanged(ULiveLinkSourceSettings* Settings, const FPropertyChangedEvent& PropertyChangedEvent) {} + virtual void ReceiveClient(ILiveLinkClient* InClient, FGuid InSourceGuid) override; + virtual bool RequestSourceShutdown(); + virtual void Update() override {} + + // custom methods + static bool GetPortGuid(int32 port, FGuid& fguid); + static bool IsValidPort(int32 port); + static FName GetConnectionName(int32 port); + static FName GetConnectionName(const FLiveLinkSubjectName& subjectName); + static FName SubjectNameFromPort(int32 port); + + void disable(); + FLiveLinkSubjectName GetSubjectName() const { return subjectKey.SubjectName; } + void SetConnectionName(FName name); + void SetHandshake(const FPoseAIHandshake& handshake); + + /* Main processing method */ + void UpdatePose(TSharedPtr jsonPose); + +private: + // We use a sharedref so that bindSP can be used to create weak references. This is only owner outside of the delegate system. + TSharedRef listener; + +public: + static const int32 portDefault = 8080; + PoseAILiveLinkServer udpServer; + +private: + /* stores ports across different sources to avoid conflict from user input */ + static TMap usedPorts; + ILiveLinkClient* liveLinkClient = nullptr; + TSharedPtr rig; + FPoseAIHandshake handshake; + int32 port; + FGuid sourceGuid ; + FLiveLinkSubjectKey subjectKey; + TUniquePtr faceSubSource; + mutable FText status; + FCriticalSection InSynchObject; + + void AddSubject(); + +}; + +/* +* This class will register for delegates as a smart pointer, allowing the owning source to only have a references from the LiveLinkClient. +*/ +class POSEAILIVELINK_API PoseAILiveLinkSingleSourceListener +{ +private: + PoseAILiveLinkNetworkSource* parent; + bool isMe(const FLiveLinkSubjectName& target) { + return target == parent->GetSubjectName(); + } +public: + PoseAILiveLinkSingleSourceListener(PoseAILiveLinkNetworkSource* parent) : parent(parent) {}; + + void SetHandshake(const FPoseAIHandshake& handshake) { + parent->SetHandshake(handshake); + } + + void CloseTarget(const FLiveLinkSubjectName& target) { + if (isMe(target)) + parent->RequestSourceShutdown(); + } + + void DisconnectTarget(const FLiveLinkSubjectName& target){ + if (isMe(target)) { + parent->udpServer.Disconnect(); + } + } + + void SendConfig(const FLiveLinkSubjectName& target, FPoseAIModelConfig config) { + if (isMe(target)) { + FString message_string = config.ToString(); + if (parent->udpServer.SendString(message_string)) + UE_LOG(LogTemp, Display, TEXT("PoseAI: Sent config %s"), *message_string); + } + } + +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h new file mode 100644 index 0000000..1819982 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkRetargetRotations.h @@ -0,0 +1,37 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once +#include "CoreMinimal.h" +#include "LiveLinkRetargetAsset.h" +#include "PoseAILiveLinkRetargetRotations.generated.h" + +// Rretarget asset for data coming from Live Link. Remaps rotations onto all bones and only translations for root and pelvis/hip. +UCLASS(Blueprintable) +class POSEAILIVELINK_API UPoseAILiveLinkRetargetRotations : public ULiveLinkRetargetAsset +{ + GENERATED_UCLASS_BODY() + + virtual ~UPoseAILiveLinkRetargetRotations() {} + + //~ Begin UObject Interface + virtual void BeginDestroy() override; + //~ End UObject Interface + + //~ Begin ULiveLinkRetargetAsset interface + virtual void BuildPoseFromAnimationData(float DeltaTime, const FLiveLinkSkeletonStaticData* InSkeletonData, const FLiveLinkAnimationFrameData* InFrameData, FCompactPose& OutPose) override; + //~ End ULiveLinkRetargetAsset interface + + // allow user to scale translations for differences in skeleton sizes + UPROPERTY(EditAnywhere, Category = Settings) + float scaleTranslation = 1.0f; + +private: + + void OnBlueprintClassCompiled(UBlueprint* TargetBlueprint); + + +#if WITH_EDITOR + /** Blueprint.OnCompiled delegate handle */ + FDelegateHandle OnBlueprintCompiledDelegate; +#endif +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h new file mode 100644 index 0000000..c6d53c7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkServer.h @@ -0,0 +1,217 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Runtime/Networking/Public/Networking.h" +#include "Runtime/Sockets/Public/Sockets.h" +#include "Runtime/Sockets/Public/SocketSubsystem.h" +#include "HAL/RunnableThread.h" +#include "LiveLinkLog.h" +#include "SocketTypes.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" +#include "IPAddress.h" +#include "Json.h" +#include "PoseAIStructs.h" +#include "PoseAIUdpSocketReceiver.h" +#include "PoseAIEndpoint.h" +#include "SocketSubsystem.h" + + +class PoseAILiveLinkReceiverRunnable; +class PoseAILiveLinkNetworkSource; +class PoseAILiveLinkServerListener; +class FPoseAISocketSender; + +// The networking class needs to be rewritten + +class POSEAILIVELINK_API PoseAILiveLinkServer +{ +public: + PoseAILiveLinkServer(FPoseAIHandshake myHandshake, bool isIPv6, int32 portNum); + void SetSource(TWeakPtr source); + + ~PoseAILiveLinkServer() { + CleanUp(); + } + + // utility function that identifies host IPv4 address, to be printed in LiveLink console to help user connect to correct address + static bool GetIP(FString& myIP); + + void CleanUp(); + void Disconnect(); + + TSharedPtr GetSocket() const { return serverSocket; } + + void ProcessNetworkPacket(const FString& recvMessage, const FPoseAIEndpoint& endpoint); + + + bool SendString(FString& message) const; + void SendHandshake() const; + void SetHandshake(const FPoseAIHandshake& handshake); + // receiver will be set on a runnable thread and set once started + void SetReceiver(TSharedPtr receiver) { udpSocketReceiver = receiver; } + + +private: + const static FString fieldPrettyName; + const static FString fieldVersion; + const static FString fieldUUID; + const static FString fieldRigType; + const static FString requiredMinVersion; + + TSharedPtr listener; + TWeakPtr source_; + FPoseAIHandshake handshake; + FName protocolType; + int32 port; + bool cleaningUp = false; + + // time of last connection. After timeout seconds a newer connection can takeover the port. + FDateTime lastConnection; + const double TIMEOUT_SECONDS = 10.0; + + TSharedPtr serverSocket; + + //used to launch receiver without slowing main thread + TSharedPtr poseAILiveLinkRunnable; + + //Listens for packets + TSharedPtr udpSocketReceiver; + + //sends instructions to paired app + TSharedPtr udpSocketSender; + FPoseAIEndpoint endpoint; + + // disconnect message formatted for Pose AI mobile app + FString disconnect = FString(TEXT("{\"REQUESTS\":[\"DISCONNECT\"]}")); + + void InitiateConnection(TSharedPtr jsonObject, const FPoseAIEndpoint& endpointRecv); + + + bool HasValidConnection() const; + + FName ExtractConnectionName(TSharedPtr jsonObject, const FPoseAIEndpoint& endpoint) const; + + // make sure mobile app is sufficiently advanced version as both endpoints of software evolve + bool CheckAppVersion(FString version) const; + + //split clean up routine by component + void CleanUpReceiver(); + void CleanUpSender(); + void CleanUpSocket(); + +}; + + + +class POSEAILIVELINK_API PoseAILiveLinkReceiverRunnable : public FRunnable +{ +public: + PoseAILiveLinkReceiverRunnable(int32 port, TSharedPtr listener, PoseAILiveLinkServer* poseAILiveLinkServer) : + port(port), poseAILiveLinkServer(poseAILiveLinkServer), listener(listener) { + myName = "PoseAILiveLinkServer_" + FGuid::NewGuid().ToString(); + thread = FRunnableThread::Create(this, *myName, 0, EThreadPriority::TPri_Normal); + } + virtual uint32 Run() override; + +protected: + FString myName; + int32 port; + FRunnableThread* thread = nullptr; +private: + PoseAILiveLinkServer* poseAILiveLinkServer; + TSharedPtr listener; + TSharedPtr udpSocketReceiver; +}; + + + + +// built in udpSocketSender kept crashing on cleanup so recreated one with sleep instead of tick/update +class POSEAILIVELINK_API FPoseAISocketSender : public FRunnable +{ +public: + FPoseAISocketSender(TSharedPtr Socket, const TCHAR* threadDescription) : + Socket(Socket) { + thread = FRunnableThread::Create(this, threadDescription, 0, EThreadPriority::TPri_Normal); + } + + + virtual uint32 Run() override { + while (running && thread == nullptr) { + FPlatformProcess::Sleep(0.2); + } + + while (running ) { + Sleep(true); + while (running && sleeping) { + FPlatformProcess::Sleep(0.005); + } + + } + + thread = nullptr; + return 0; + } + + virtual void Stop() override { + running = false; + if (thread != nullptr) { + Sleep(false); + } + } + + bool Send(const TSharedRef, ESPMode::ThreadSafe>& Data, const FPoseAIEndpoint& Recipient) + { + if (running) { + + int32 sent = 0; + if (!Socket) { + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: socket missing from sender")); + return false; + } + + if (!Socket->SendTo(Data->GetData(), Data->Num(), sent, *Recipient.ToInternetAddr())) + UE_LOG(LogTemp, Warning, TEXT("PoseAI LiveLink: unable to send to %s"), *(Recipient.ToString())); + + if (sent != Data->Num()) + return false; + + Sleep(false); + return true; + } + return false; + } + + void Sleep(bool sleep) { + sleeping = sleep; + if (thread != nullptr) + thread->Suspend(sleep); + } + +protected: + /** The network socket. */ + TSharedPtr Socket; + + /** The thread object. */ + FRunnableThread* thread = nullptr; + + bool running = true; + bool sleeping = false; +}; + + +/* +* To improve stability with the delegate system we use a listener component class which +* can be wrapped with smart pointers for binding (raw pointer delegate bindings are a potential source of crashes) +*/ +class PoseAILiveLinkServerListener { +public: + void ReceiveUDPDelegate(const FString& recvMessage, const FPoseAIEndpoint& endpoint) { + parent->ProcessNetworkPacket(recvMessage, endpoint); + } + PoseAILiveLinkServerListener(PoseAILiveLinkServer* parent) : parent(parent) {} +private: + PoseAILiveLinkServer* parent; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h new file mode 100644 index 0000000..73f462b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAILiveLinkSourceFactory.h @@ -0,0 +1,30 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "LiveLinkSourceFactory.h" +#include "ILiveLinkSource.h" +#include "PoseAILiveLinkSourceFactory.generated.h" + +/** + * + */ +UCLASS() +class POSEAILIVELINK_API UPoseAILiveLinkSourceFactory : public ULiveLinkSourceFactory +{ + GENERATED_BODY() + + UPoseAILiveLinkSourceFactory() { + } + + ~UPoseAILiveLinkSourceFactory () { + } + +public: + virtual TSharedPtr< SWidget > BuildCreationPanel(FOnLiveLinkSourceCreated OnLiveLinkSourceCreated) const override; + virtual TSharedPtr< ILiveLinkSource > CreateSource(const FString& ConnectionString) const override; + virtual FText GetSourceDisplayName() const override; + virtual FText GetSourceTooltip() const override; + virtual EMenuType GetMenuType() const override { return EMenuType::SubPanel; } +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h new file mode 100644 index 0000000..1e8e6ac --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIRig.h @@ -0,0 +1,151 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "GenericPlatform/GenericPlatformMath.h" +#include "ILiveLinkSource.h" +#include "LiveLinkSubjectSettings.h" +#include "Roles/LiveLinkAnimationTypes.h" +#include "Json.h" +#include "PoseAIStructs.h" + +struct POSEAILIVELINK_API Remapping +{ + FName TargetJointName; + FQuat RotAdj; + Remapping(FName TargetJointName, FQuat RotAdj) : TargetJointName(TargetJointName), RotAdj(RotAdj) {}; +}; + + + +/** + * Abstract base class for the different rig formats streamable by Pose AI + */ +class POSEAILIVELINK_API PoseAIRig +{ + public: + FLiveLinkStaticDataStruct MakeStaticData(); + bool ProcessFrame(const TSharedPtr, FLiveLinkAnimationFrameData& data); + static bool IsFrameData(const TSharedPtr jsonObject); + static TSharedPtr PoseAIRigFactory(const FLiveLinkSubjectName& name, const FPoseAIHandshake& handshake); + static TWeakPtr GetRigFromSubjectName(const FLiveLinkSubjectName& name); + FName RigType() { return rigType; } + + FPoseAIVisibilityFlags visibilityFlags; + FPoseAILiveValues liveValues; + FPoseAIScalarStruct scalars; + FPoseAIEventStruct events; + + bool useNextRootAsOffset = false; + //ankle to head top height for scaling PoseAI root motion. + float rigHeight = 170.0f; + + float CameraTilt = 0.0f; + + protected: + FLiveLinkStaticDataStruct rig; + FPoseAIVerbose verbose; + static const FString fieldBody; + static const FString fieldRigType; + static const FString fieldHandLeft; + static const FString fieldHandRight; + static const FString fieldRotations; + static const FString fieldEvents; + static const FString fieldScalars; + static const FString fieldVectors; + + protected: + PoseAIRig(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake); + virtual ~PoseAIRig() { + }; + + /* sets up skeletal heirarchy and provides default locations for each transform based on default skeleton (i.e. UE4 Mannequen or male Metahuman). + These bone lengths ensure a sensible animation is created even if user does not retarget from livelink */ + virtual void Configure(); //impure for MacOS compatibility + + FLiveLinkSubjectName name; + FName rigType; + + bool includeHands; + bool isMirrored; + bool isLowerBodyRotated; + bool isDesktop; + int32 numBodyJoints = 21; + int32 numHandJoints = 17; + // number of joints to insert in desktop mode (as camera omits quaternions for unused joints) + int32 lowerBodyNumOfJoints = 8; + + int32 rShinJoint = 3; + int32 lShinJoint = 7; + + bool isCrouching = false; + int32 handZoneL = 5; + int32 handZoneR = 5; + int32 stableFeet = 0; + FVector prevRootTranslation = FVector::ZeroVector; + // store translations of deployed rig + TMap boneVectors; + TArray jointNames; + TArray parentIndices; + TArray cachedPose = {}; + + //temporary variable used for convenience in rig construction + FName lastBoneAdded; + + //extra offset for hip bone to accomodate mesh thickness from bone sockets. + float rootHipOffsetZ = 2.0f; + + void AddBone(FName boneName, FName parentName, FVector translation); + void AddBoneToLast(FName boneName, FVector translation); + void CachePose(const TArray& transforms); + void AppendQuatArray(const TArray& quatArray, int32 begin, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AppendCachedRotations(int32 begin, int32 end, TArray& componentRotations, FLiveLinkAnimationFrameData& data); + void AssignCharacterMotion(FLiveLinkAnimationFrameData& data); + bool ProcessVerboseRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + bool ProcessCompactRotations(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessVerboseSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void ProcessCompactSupplementaryData(const TSharedPtr jsonObject, FLiveLinkAnimationFrameData& data); + void TriggerEvents(); + void RotateLowerBody180(TArray& quatArray); + + +private: + static TMap> RigMap; +}; + +class POSEAILIVELINK_API PoseAIRigUE4 : public PoseAIRig { + public: + PoseAIRigUE4(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamo : public PoseAIRig { + public: + PoseAIRigMixamo(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigMixamoAlt : public PoseAIRig { +public: + PoseAIRigMixamoAlt(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + + +class POSEAILIVELINK_API PoseAIRigMetaHuman : public PoseAIRig { +public: + PoseAIRigMetaHuman(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; + +class POSEAILIVELINK_API PoseAIRigDazUE : public PoseAIRig { +public: + PoseAIRigDazUE(FLiveLinkSubjectName name, const FPoseAIHandshake& handshake) : PoseAIRig(name, handshake) {}; +protected: + void Configure(); +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h new file mode 100644 index 0000000..32ae5d2 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIStructs.h @@ -0,0 +1,529 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Json.h" +#include "JsonObjectConverter.h" +#include "PoseAIStructs.generated.h" + + + +/* decoding utilities for compact representation */ +float UintB64ToUint(char a, char b); +uint32 UintB64ToUint(char a, char b, char c); +float FixedB64pairToFloat(char a, char b); +void FStringFixed12ToFloat(const FString& data, TArray& flatArray); +void FlatArrayToQuats(const TArray& flatArray, TArray& quatArray); + +UENUM(BlueprintType) +enum class EPoseAiPacketFormat : uint8 +{ + Verbose, Compact +}; + +UENUM(BlueprintType) +enum class EPoseAiAppModes : uint8 +{ + Room, Desktop, Portrait, RoomBodyOnly, PortraitBodyOnly +}; + +UENUM(BlueprintType) +enum class EPoseAiContext : uint8 +{ + Default +}; + +UENUM(BlueprintType) +enum class EPoseAiRigPresets : uint8 +{ + MetaHuman, UE4, Mixamo, DazUE, MixamoAlt +}; +UENUM(BlueprintType) +enum class EPoseAiHandModel : uint8 +{ + Version1, Version2_EXPERIMENTAL +}; +UENUM(BlueprintType) +enum class EPoseAiBodyModel : uint8 +{ + Version2, Version3 +}; + + + +/* the handshake configures the main parameters of pose camera*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIHandshake +{ + GENERATED_BODY() + + /* the camera mode. "Room", "Desktop", "Portrait", "RoomBodyOnly", "PortraitBodyOnly" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiAppModes mode = EPoseAiAppModes::Room; + + /* the skeletal rig to use, based on standard nomenclature and rotations: "UE4", "MetaHuman", "DazUE", "Mixamo" */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiRigPresets rig = EPoseAiRigPresets::MetaHuman; + + /* BETA: provides ARKit compatible animation blendshape stream for facial rigs */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isFaceAnimating = true; + + /* flips left/right limbs and rotates as if the player is looking at a mirror*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isMirrored = true; + + /* rotates lower body 180 degrees - convenient for desktop mode in some perspectives*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool isLowerBodyRotated = false; + + /* the desired camera speed. On many phones only 30 or 60 FPS will be accepted and otherwise you get default*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 cameraFPS = 60; + + /* target frame rate for phone interpolation smoothing. Suggest 0 on Unreal. Events are raw.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + int32 syncFPS = 0; + + /* version of our AI model: V2 is 2022 release, V3 currently Room/Portrait mode only as of March 2023 release*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiBodyModel bodyModelVersion = EPoseAiBodyModel::Version2; + + /* the version of our hand AI. Version 1 is our original. Version 2 is experimental and may offer some improvments.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiHandModel handModelVersion = EPoseAiHandModel::Version1; + + /* the model context. Reserved for future AI models*/ + UPROPERTY(EditAnywhere, Category = "PoseAI Handshake") + EPoseAiContext context = EPoseAiContext::Default; + + /* Not needed for PoseCam. Used only for licensee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString whoami = ""; + + /* Not needed for PoseCam. Used only for licencee connection and verification.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + FString signature = ""; + + /* Turn on demo locomotion / action recognition events. Keep off for efficiency unless testing.*/ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + bool locomotionEvents = false; + + /* controls compactness of packet. */ + UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "PoseAI Handshake") + EPoseAiPacketFormat packetFormat = EPoseAiPacketFormat::Compact; + + + bool operator==(const FPoseAIHandshake& Other) const; + bool operator!=(const FPoseAIHandshake& Other) const { return !operator==(Other); } + + bool IncludesHands() const; + FString GetContextString() const; + FString GetModeString() const; + FString GetRigString() const; + int32 GetBodyModelVersion() const; + int32 GetHandModelVersion() const; + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + + +/*adjusts the sensitivity of PoseAI events*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIModelConfig +{ + GENERATED_BODY() + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float stepSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float armSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float crouchSensitivity = 0.5f; + + /* alpha where 0.0 is lowest sensitivity (more likely to miss events) and 1.0 is maximum sensitivity (more likely false triggers).*/ + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + float jumpSensitivity = 0.5f; + + UPROPERTY(BlueprintReadWrite, Category = "PoseAI Model Sensitivity") + bool isMirrored = true; + + FString ToString() const; + FString YesNoString(bool val) const { + return val ? FString("YES") : FString("NO"); + } +}; + + +/** base class for the two event notifications formats sent by camera. All events have a uint count, which upon change signifies a new event has been registered + and a second property, either a float or uint. +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /** number of events registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Count = 0; + + virtual void ProcessCompact(const FString& compactString) {}; + bool CheckTriggerAndUpdate(); +private: + uint32 InternalCount = 0; +}; + +/** +*structure to hold a type of event notification +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventPair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**magnitude of event where approrpiate */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + float Magnitude = 0.0f; + + void ProcessCompact(const FString& compactString) override; + +}; + + +USTRUCT() +struct POSEAILIVELINK_API FPoseAIGesturePair : public FPoseAIEventPairBase +{ + GENERATED_BODY() +public: + /**index code for most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI Event") + uint32 Current = 0; + + void ProcessCompact(const FString& compactString) override; + +}; + + +/** +*structure to receive event notifications +*/ +USTRUCT() +struct POSEAILIVELINK_API FPoseAIEventStruct +{ + GENERATED_BODY() +public: + /** number of footsteps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Footstep; + + /** number of left foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepL; + + /** number of right foot sidesteps registered by camera. sign indicates direction */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair SidestepR; + + /** number of jumps registered by camera */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair Jump; + + /** number of footsplits registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair FeetSplit; + + /** number of arm pumps registered by camera, body height magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmPump; + + /** number of arm flexes registered by camera, body width magnitude */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIEventPair ArmFlex; + + /** number of left or dual arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureL; + + /** number of right arm gestures registered by camera and most recent gesture */ + UPROPERTY(VisibleAnywhere, Category = "PoseAI") + FPoseAIGesturePair ArmGestureR; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > eveBody); + void ProcessCompactBody(const FString& compactString); + +}; + +/** +*structure to share additional information +*/ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIScalarStruct +{ + GENERATED_BODY() +public: + /** visibility flags by body part. Correspond to figure in the app */ + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisTorso = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisArmR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegL = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisLegR = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + float VisFace = 0.0f; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 HandZoneL = 5; + + /** location of right hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PosPeAI") + int32 HandZoneR = 5; + + /** Heading in degrees of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float ChestYaw = 0.0f; + + /** Heading in degrees of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float StanceYaw = 0.0f; + + /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float BodyHeight = 0.0f; + + /** whether subject is crouching */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float IsCrouching = 0.0f; + + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 StableFoot = 0.0f; + + + void ProcessJsonObject(const TSharedPtr < FJsonObject > scaBody); + +}; + + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerboseBodyVectors +{ + GENERATED_BODY() +public: + UPROPERTY() + TArray HipLean; + UPROPERTY() + TArray HipScreen; + UPROPERTY() + TArray ChestScreen; + UPROPERTY() + TArray HandIkL; + UPROPERTY() + TArray HandIkR; + UPROPERTY() + TArray Hip; + UPROPERTY() + TArray FootIkL; + UPROPERTY() + TArray FootIkR; +}; + + +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVerbose +{ + GENERATED_BODY() +public: + UPROPERTY() + FPoseAIEventStruct Events; + UPROPERTY() + FPoseAIScalarStruct Scalars; + UPROPERTY() + FPoseAIVerboseBodyVectors Vectors; + void ProcessJsonObject(const TSharedPtr < FJsonObject > jsonObj); + +}; + +/** + * structure to store and expose visibility flags for events alerting programmer if subject is out of camera + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAIVisibilityFlags +{ + GENERATED_BODY() +public: + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isTorso = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightArm = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isLeftLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isRightLeg = false; + + UPROPERTY(BlueprintReadOnly, Category = "Flags") + bool isFace = false; + + bool HasChanged() { return hasChanged; } + void ProcessVerbose(FPoseAIScalarStruct& scalars); + void ProcessCompact(const FString& visString); + +private: + bool hasChanged = false; +}; + +/** + * structure to share additional information + */ +USTRUCT(BlueprintType) +struct POSEAILIVELINK_API FPoseAILiveValues +{ +GENERATED_BODY() +public: + + /** How much subject is leaning in radians, head-to-hips; x to the side, y forward */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D upperBodyLean = FVector2D(0.0f, 0.0f); + + /** estimated stance height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + bool isCrouching = false; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneLeft = 5; + + /** location of left hand relative to body in broad zones */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 handZoneRight = 5; + + /** location of subject in camera frame, scaled in body units (i.e. multiply by rig height to translate to game world). Pos Y moves toward camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector rootTranslation = FVector(0.0f, -2.0f, 0.0f); + + /** Heading in radians of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float stanceYaw = 0.0f; + + /** Heading in radians of torso. 0 is heading to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float chestYaw = 0.0f; + + /** Heading of flattened left foot to right foot vector relative to camera */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 modelLatency = 0.0f; + + /** timestamp according to pose camera device (CMTime), in seconds */ + double timestamp = 0.0; + + /** DEPR current height of jump in body units */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI_DEPR") + float jumpHeight = 0.0f; + + /** DEPR estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI_DEPR") + float bodyHeight = 0.0f; + + /** location of hips in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D hipScreen = FVector2D(0.0f, 0.0f); + + /** location of chest in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D chestScreen = FVector2D(0.0f, 0.0f); + + /** location of left index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandLeft = FVector2D(0.0f, 0.0f); + + /** location of right index finger in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category="PoseAI") + FVector2D pointHandRight = FVector2D(0.0f, 0.0f); + + /** location of left thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbLeft = FVector2D(0.0f, 0.0f); + + /** location of right thumb tip in camera frame (clip coordinates -1 to 1, 0 is center ) */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector2D pointThumbRight = FVector2D(0.0f, 0.0f); + + /** how open the left hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessLeftHand = 0.5f; + + /** how open the right hand is currently */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + float opennessRightHand = 0.5f; + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector handIkR = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkL = FVector(0.0f, 0.0f, 0.0f); + + /** target for the PoseAI Foot IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector footIkR = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkL = FVector(0.0f, 0.0f, 0.0f); + + /** finger target for the PoseAI Hand IK node */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + FVector fingerIkR = FVector(0.0f, 0.0f, 0.0f); + + /** if at least one foot has been stationary for a few frames */ + UPROPERTY(BlueprintReadOnly, Category = "PoseAI") + int32 stableFeet = 0; + + FVector rootOffset = FVector(0.0f, 0.0f, 0.0f); + FRotator cameraRotation = FRotator(0.0f, 0.0f, 0.0f); + FVector scaleMotion = FVector(1.0f, 0.0f, 1.0f); + + void ProcessVerboseBody(const FPoseAIVerbose& scalars); + void ProcessVerboseVectorsHandLeft(const TSharedPtr < FJsonObject > vecHand); + void ProcessVerboseVectorsHandRight(const TSharedPtr < FJsonObject > vecHand); + void ProcessCompactScalarsBody(const FString& compactString); + void ProcessCompactVectorsBody(const FString& compactString); + void ProcessCompactVectorsHandLeft(const TSharedPtr < FJsonObject >); + void ProcessCompactVectorsHandRight(const TSharedPtr < FJsonObject >); + +private: + static const FString fieldPointScreen; + static const FString fieldThumbScreen; + +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h new file mode 100644 index 0000000..9434a67 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/PoseAIUdpSocketReceiver.h @@ -0,0 +1,220 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. +// This is a minor edit of Epic Games FUdpSocetReceiver class to allow different protocols (like IPv6) + +#pragma once + +#include "CoreMinimal.h" +#include "HAL/Runnable.h" +#include "HAL/RunnableThread.h" +#include "Misc/SingleThreadRunnable.h" +#include "Serialization/ArrayReader.h" +#include "Sockets.h" +#include "SocketSubsystem.h" +#include "Interfaces/IPv4/IPv4Endpoint.h" + +#include "PoseAIEndpoint.h" +#include "IPAddress.h" + + + + + +/** + * Temporary fix for concurrency crashes. This whole class will be redesigned. + */ +typedef TSharedPtr FArrayReaderPtr; + +/** + * Delegate type for received data. + * + * The first parameter is the received data. + * The second parameter is sender's IP endpoint. + */ +DECLARE_DELEGATE_TwoParams(FPoseAIOnSocketDataReceived, const FString&, const FPoseAIEndpoint&); //Change delegate name and use our endpoint + + +/** + * Asynchronously receives data from an UDP socket. + */ +class FPoseAIUdpSocketReceiver + : public FRunnable + , private FSingleThreadRunnable +{ +public: + + /** + * Creates and initializes a new socket receiver. + * + * @param InSocket The UDP socket to receive data from. + * @param InWaitTime The amount of time to wait for the socket to be readable. + * @param InThreadName The receiver thread name (for debugging). + */ + FPoseAIUdpSocketReceiver(TSharedPtr InSocket, const FTimespan& InWaitTime, const TCHAR* InThreadName) + : Socket(InSocket) + , Stopping(false) + , Thread(nullptr) + , ThreadName(InThreadName) + , WaitTime(InWaitTime) + { + check(Socket != nullptr); + check(Socket->GetSocketType() == SOCKTYPE_Datagram); + Reader->SetNumUninitialized(MaxReadBufferSize); + SocketSubsystem = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM); + } + + /** Virtual destructor. */ + virtual ~FPoseAIUdpSocketReceiver() + { + if (Thread != nullptr) + { + Thread->Kill(true); + delete Thread; + } + } + +public: + + /** Set the maximum size allocated to read off of the socket. */ + void SetMaxReadBufferSize(uint32 InMaxReadBufferSize) + { + MaxReadBufferSize = InMaxReadBufferSize; + } + + /** Start the receiver thread. */ + void Start() + { + Thread = FRunnableThread::Create(this, *ThreadName, 128 * 1024, TPri_AboveNormal, FPlatformAffinity::GetPoolThreadMask()); + } + + /** + * Returns a delegate that is executed when data has been received. + * + * This delegate must be bound before the receiver thread is started with + * the Start() method. It cannot be unbound while the thread is running. + * + * @return The delegate. + */ + FPoseAIOnSocketDataReceived& OnDataReceived() + { + check(Thread == nullptr); + return DataReceivedDelegate; + } + +public: + + //~ FRunnable interface + + virtual FSingleThreadRunnable* GetSingleThreadInterface() override + { + return this; + } + + virtual bool Init() override + { + return true; + } + + virtual uint32 Run() override + { + while (!Stopping) + { + isUpdating = true; + Update(WaitTime); + isUpdating = false; + } + + return 0; + } + + virtual void Stop() override + { + Stopping = true; + } + + virtual void Exit() override { } + +protected: + + /** Update this socket receiver. */ + void Update(const FTimespan& SocketWaitTime) + { + + if (!Socket->Wait(ESocketWaitConditions::WaitForRead, SocketWaitTime)) + { + return; + } + + /************************************************* + Hwere we make changes to specify address with protocol + **********************************************************/ + if (Stopping) + return; + + TSharedRef Sender = SocketSubsystem->CreateInternetAddr(Socket->GetProtocol()); + uint32 Size; + while (Socket && Socket.IsValid() && Socket->HasPendingData(Size)) + { + // we also send the messages via delegate as FStrings instead of FArrayReaderPtrs + + int32 BytesRead = 0; + if (Socket->RecvFrom(Reader->GetData(), FMath::Min(Size, MaxReadBufferSize), BytesRead, *Sender)) + { + + // UE4.2x versions + //UTF8CHAR* bytedata_utf8 = (UTF8CHAR*)Reader->GetData(); + //TCHAR* bytedata = UTF8_TO_TCHAR(bytedata_utf8); + // end UE4.2x + + // UE5.0 + UTF8CHAR* bytedata = (UTF8CHAR*)Reader->GetData(); + // end UE5.0 + + FString recvMessage = FString(BytesRead, bytedata); + DataReceivedDelegate.ExecuteIfBound(recvMessage, FPoseAIEndpoint(Sender)); + } + + } + + + } + +protected: + + //~ FSingleThreadRunnable interface + + virtual void Tick() override + { + Update(FTimespan::Zero()); + } + +private: + FArrayReaderPtr Reader = MakeShared(true); + /** The network socket. */ + TSharedPtr Socket = nullptr; + + /** Pointer to the socket sub-system. */ + ISocketSubsystem* SocketSubsystem = nullptr; + + /** Flag indicating that the thread is stopping. */ + bool Stopping; + + /** The thread object. */ + FRunnableThread* Thread = nullptr; + + /** The receiver thread's name. */ + FString ThreadName; + + /** The amount of time to wait for inbound packets. */ + FTimespan WaitTime; + + /** The maximum read buffer size used to read the socket. */ + uint32 MaxReadBufferSize = 65507u; + + bool isUpdating = false; + +private: + + /** Holds the data received delegate. */ + FPoseAIOnSocketDataReceived DataReceivedDelegate; +}; + diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h new file mode 100644 index 0000000..dfa1571 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/SPoseAILiveLinkWidget.h @@ -0,0 +1,78 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "CoreGlobals.h" +#include "Widgets/SCompoundWidget.h" +#include "Widgets/SBoxPanel.h" +#include "Widgets/Input/SEditableTextBox.h" +#include "Widgets/Text/STextBlock.h" +#include "Widgets/Input/SButton.h" +#include "Widgets/Input/SCheckBox.h" +#include "Widgets/Layout/SBox.h" +#include "Types/WidgetActiveTimerDelegate.h" +#include "SlateOptMacros.h" +#include "Misc/ConfigCacheIni.h" +#include "LiveLinkLog.h" +#include "iLiveLinkSource.h" +#include "LiveLinkSourceFactory.h" +#include "IPAddress.h" +#include "PoseAIStructs.h" + + +/** + * + */ +class POSEAILIVELINK_API SPoseAILiveLinkWidget : public SCompoundWidget, public FWidgetActiveTimerDelegate +{ +public: + SLATE_BEGIN_ARGS(SPoseAILiveLinkWidget) {} + SLATE_END_ARGS() + + /** Constructs this widget with InArgs */ + void Construct(const FArguments& InArgs); + + void setCallback(ULiveLinkSourceFactory::FOnLiveLinkSourceCreated whenCreated) { callback = whenCreated; } + + static void disableExistingSource(); + static TSharedPtr CreateSource(const FString& port); + +protected: + static TWeakPtr source; + ULiveLinkSourceFactory::FOnLiveLinkSourceCreated callback; + + UPROPERTY(EditAnywhere, Config, Category = Custom) + +private: + const static FString section; + static int32 portNum; + static int32 syncFPS; + static int32 cameraFPS; + static int32 modeIndex; + static int32 rigIndex; + static bool isMirrored; + static bool isIPv6; + + static FPoseAIHandshake GetHandshake(); + void UpdatePort(const FText& InText, ETextCommit::Type type); + void UpdateSyncFPS(const FText& InText, ETextCommit::Type type); + void UpdateCameraFPS(const FText& InText, ETextCommit::Type type); + bool IsPortValid() const; + + FReply OnOkClicked(); + FReply OnToggleModeClicked(); + FReply OnToggleRigClicked(); + + TWeakPtr ipv6CheckBox = nullptr; + TSharedPtr portInput = nullptr; + TSharedPtr syncFpsInput = nullptr; + TSharedPtr cameraFpsInput = nullptr; + TSharedPtr modeInput = nullptr; + TSharedPtr rigInput = nullptr; + TWeakPtr mirroredCheckBox = nullptr; + TWeakPtr mixamoCheckBox = nullptr; + TWeakPtr rootMotionCheckBox = nullptr; + + void ReadCheckBox(TWeakPtr& checkBox, bool& readTo); +}; \ No newline at end of file diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini new file mode 100644 index 0000000..d957fd1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLink/Public/desktop.ini @@ -0,0 +1,4 @@ +[ViewState] +Mode= +Vid= +FolderType=Generic diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs new file mode 100644 index 0000000..e39bf7b --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/PoseAILiveLinkEd.Build.cs @@ -0,0 +1,60 @@ +// Copyright 2022 Pose AI Ltd. All Rights Reserved. + +using UnrealBuildTool; + +public class PoseAILiveLinkEd : ModuleRules +{ + public PoseAILiveLinkEd(ReadOnlyTargetRules Target) : base(Target) + { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; + + PublicIncludePaths.AddRange( + new string[] { + // ... add public include paths required here ... + } + ); + + + PrivateIncludePaths.AddRange( + new string[] { + // ... add other private include paths required here ... + } + ); + + + PublicDependencyModuleNames.AddRange( + new string[] + { + "Core", + "CoreUObject", + "Engine", + "InputCore", + "AnimationCore", + "AnimGraphRuntime", + // ... add other public dependencies that you statically link with here ... + } + ); + + + PrivateDependencyModuleNames.AddRange( + new string[] + { + "Slate", + "SlateCore", + "AnimGraph", + "PoseAILiveLink", + "BlueprintGraph", // to be checked if this is an issue for packaging + + // ... add private dependencies that you statically link with here ... + } + ); + + + DynamicallyLoadedModuleNames.AddRange( + new string[] + { + // ... add any modules that your module loads dynamically here ... + } + ); + } +} diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp new file mode 100644 index 0000000..11756e1 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIGroundPenetration.cpp @@ -0,0 +1,124 @@ +// Copyright 2022-2023 Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIGroundPenetration.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// + +class FPoseAIGroundPenetrationDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIGroundPenetration::PoseAIGroundPenetrationDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIGroundPenetration + + +UAnimGraphNode_PoseAIGroundPenetration::UAnimGraphNode_PoseAIGroundPenetration(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetControllerDescription() const +{ + return LOCTEXT("PoseAIGroundPenetration", "PoseAI Ground Penetration"); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Tooltip", "This control makes sure avatar doesn't penetrate bottom of capsule, and can also pin the avatar lowpoint to capsule floor."); +} + +FText UAnimGraphNode_PoseAIGroundPenetration::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.BoneToModify.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.BoneToModify.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_PoseAIGroundPenetration_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIGroundPenetration* PoseAIGroundPenetration = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + +} + +void UAnimGraphNode_PoseAIGroundPenetration::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIGroundPenetrationDelegate.IsValid()) + { + PoseAIGroundPenetrationDelegate = MakeShareable(new FPoseAIGroundPenetrationDelegate()); + } + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + +} + +void UAnimGraphNode_PoseAIGroundPenetration::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIGroundPenetration* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + //pass + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp new file mode 100644 index 0000000..4a24dda --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/AnimGraphNode_PoseAIHandTarget.cpp @@ -0,0 +1,195 @@ +// Copyright Pose AI Ltd. All Rights Reserved. + +#include "AnimGraphNode_PoseAIHandTarget.h" +#include "AnimNodeEditModes.h" +#include "Animation/AnimInstance.h" + +// for customization details +#include "PropertyHandle.h" +#include "DetailLayoutBuilder.h" +#include "DetailCategoryBuilder.h" + +// version handling +#include "AnimationCustomVersion.h" +#include "UObject/ReleaseObjectVersion.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +///////////////////////////////////////////////////// +// FTwoBoneIKDelegate + +class FPoseAIHandTargetDelegate : public TSharedFromThis +{ +public: + void UpdateLocationSpace(class IDetailLayoutBuilder* DetailBuilder) + { + if (DetailBuilder) + { + DetailBuilder->ForceRefreshDetails(); + } + } +}; + +TSharedPtr UAnimGraphNode_PoseAIHandTarget::PoseAIHandTargetDelegate = NULL; + +///////////////////////////////////////////////////// +// UAnimGraphNode_PoseAIHandTarget + + +UAnimGraphNode_PoseAIHandTarget::UAnimGraphNode_PoseAIHandTarget(const FObjectInitializer& ObjectInitializer) + : Super(ObjectInitializer) +{ +} + +FText UAnimGraphNode_PoseAIHandTarget::GetControllerDescription() const +{ + return LOCTEXT("PoseAIHandTarget", "PoseAI Hands In BodySpace"); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetTooltipText() const +{ + return LOCTEXT("AnimGraphNode_PoseAIHandTarget_Tooltip", "ThIS control applies an inverse kinematic (IK) solver to a 3-joint chain, based on remapped coordinates between different sized avatars."); +} + +FText UAnimGraphNode_PoseAIHandTarget::GetNodeTitle(ENodeTitleType::Type TitleType) const +{ + if ((TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) && (Node.IKBone.BoneName == NAME_None)) + { + return GetControllerDescription(); + } + // @TODO: the bone can be altered in the property editor, so we have to + // choose to mark this dirty when that happens for this to properly work + else //if (!CachedNodeTitles.IsTitleCached(TitleType, this)) + { + FFormatNamedArguments Args; + Args.Add(TEXT("ControllerDescription"), GetControllerDescription()); + Args.Add(TEXT("BoneName"), FText::FromName(Node.IKBone.BoneName)); + + // FText::Format() is slow, so we cache this to save on performance + if (TitleType == ENodeTitleType::ListView || TitleType == ENodeTitleType::MenuTitle) + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_ListTitle", "{ControllerDescription} - Bone: {BoneName}"), Args), this); + } + else + { + CachedNodeTitles.SetCachedTitle(TitleType, FText::Format(LOCTEXT("AnimGraphNode_IKBone_Title", "{ControllerDescription}\nBone: {BoneName}"), Args), this); + } + } + return CachedNodeTitles[TitleType]; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) +{ + FAnimNode_PoseAIHandTarget* PoseAIHandTarget = static_cast(InPreviewNode); + + // copies Pin values from the internal node to get data which are not compiled yet + PoseAIHandTarget->PoseAiIkVector = Node.PoseAiIkVector; +} + +void UAnimGraphNode_PoseAIHandTarget::CopyPinDefaultsToNodeData(UEdGraphPin* InPin) +{ + if (InPin->GetName() == GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector)) + { + GetDefaultValue(GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, PoseAiIkVector), Node.PoseAiIkVector); + } +} + +void UAnimGraphNode_PoseAIHandTarget::CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) +{ + Super::Super::CustomizeDetails(DetailBuilder); + + // initialize just once + if (!PoseAIHandTargetDelegate.IsValid()) + { + PoseAIHandTargetDelegate = MakeShareable(new FPoseAIHandTargetDelegate()); + } + + + IDetailCategoryBuilder& IKCategory = DetailBuilder.EditCategory("IK"); + IDetailCategoryBuilder& EffectorCategory = DetailBuilder.EditCategory("Effector"); + IDetailCategoryBuilder& JointCategory = DetailBuilder.EditCategory("JointTarget"); + + + EBoneControlSpace Space = Node.EffectorLocationSpace; + const FString TakeRotationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, bTakeRotationFromEffectorSpace)); + const FString EffectorTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorTarget)); + const FString EffectorLocationPropName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocation)); + const FString EffectorLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, EffectorLocationSpace)); + // hide all properties in EndEffector category + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*TakeRotationPropName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*EffectorLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + //Space = Node.JointTargetLocationSpace; + bool bPinVisibilityChanged = false; + const FString JointTargetName = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTarget)); + const FString JointTargetLocation = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocation)); + const FString JointTargetLocationSpace = FString::Printf(TEXT("Node.%s"), GET_MEMBER_NAME_STRING_CHECKED(FAnimNode_PoseAIHandTarget, JointTargetLocationSpace)); + + // hide all properties in JointTarget category except for JointTargetLocationSpace + { + TSharedPtr PropertyHandle = DetailBuilder.GetProperty(*JointTargetName, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocationSpace, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + PropertyHandle = DetailBuilder.GetProperty(*JointTargetLocation, GetClass()); + DetailBuilder.HideProperty(PropertyHandle); + } + + +} + +void UAnimGraphNode_PoseAIHandTarget::Serialize(FArchive& Ar) +{ + Super::Serialize(Ar); + + Ar.UsingCustomVersion(FAnimationCustomVersion::GUID); + + const int32 CustomAnimVersion = Ar.CustomVer(FAnimationCustomVersion::GUID); + + if (CustomAnimVersion < FAnimationCustomVersion::RenamedStretchLimits) + { + // fix up deprecated variables + Node.StartStretchRatio = Node.StretchLimits_DEPRECATED.X; + Node.MaxStretchScale = Node.StretchLimits_DEPRECATED.Y; + } + + Ar.UsingCustomVersion(FReleaseObjectVersion::GUID); + if (Ar.CustomVer(FReleaseObjectVersion::GUID) < FReleaseObjectVersion::RenameNoTwistToAllowTwistInTwoBoneIK) + { + Node.bAllowTwist = !Node.bNoTwist_DEPRECATED; + } + + if (CustomAnimVersion < FAnimationCustomVersion::ConvertIKToSupportBoneSocketTarget) + { + if (Node.EffectorSpaceBoneName_DEPRECATED != NAME_None) + { + Node.EffectorTarget = FBoneSocketTarget(Node.EffectorSpaceBoneName_DEPRECATED); + } + + if (Node.JointTargetSpaceBoneName_DEPRECATED != NAME_None) + { + Node.JointTarget = FBoneSocketTarget(Node.JointTargetSpaceBoneName_DEPRECATED); + } + } +} + +void UAnimGraphNode_PoseAIHandTarget::Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const +{ + if (bEnableDebugDraw && SkelMeshComp) + { + if (FAnimNode_PoseAIHandTarget* ActiveNode = GetActiveInstanceNode(SkelMeshComp->GetAnimInstance())) + { + ActiveNode->ConditionalDebugDraw(PDI, SkelMeshComp); + } + } +} + +#undef LOCTEXT_NAMESPACE diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp new file mode 100644 index 0000000..df1d03d --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Private/PoseAILiveLinkEd.cpp @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#include "PoseAILiveLinkEd.h" +#include "Core.h" +#include "Interfaces/IPluginManager.h" + +#define LOCTEXT_NAMESPACE "PoseAI" + +void FPoseAILiveLinkEdModule::StartupModule() +{ + +} + +void FPoseAILiveLinkEdModule::ShutdownModule() +{ + +} + +//#undef LOCTEXT_NAMESPACE + +IMPLEMENT_MODULE(FPoseAILiveLinkEdModule, PoseAILiveLinkEd) diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h new file mode 100644 index 0000000..aeed4a7 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIGroundPenetration.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022-2023. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIGroundPenetration.h" +#include "AnimGraphNode_PoseAIGroundPenetration.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIGroundPenetrationDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIGroundPenetration : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIGroundPenetration Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIGroundPenetrationDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h new file mode 100644 index 0000000..0487a75 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/AnimGraphNode_PoseAIHandTarget.h @@ -0,0 +1,66 @@ +// Copyright Pose AI 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/ObjectMacros.h" +#include "Engine/TargetPoint.h" +#include "EdGraph/EdGraphNodeUtils.h" +#include "AnimGraphNode_SkeletalControlBase.h" +#include "AnimNode_PoseAIHandTarget.h" +#include "AnimGraphNode_PoseAIHandTarget.generated.h" + +// actor class used for bone selector +#define ABoneSelectActor ATargetPoint + +class FPoseAIHandTargetDelegate; +class IDetailLayoutBuilder; + +UCLASS(MinimalAPI) +class UAnimGraphNode_PoseAIHandTarget : public UAnimGraphNode_SkeletalControlBase +{ + GENERATED_UCLASS_BODY() + + UPROPERTY(EditAnywhere, Category=Settings) + FAnimNode_PoseAIHandTarget Node; + + /** Enable drawing of the debug information of the node */ + UPROPERTY(EditAnywhere, Category=Debug) + bool bEnableDebugDraw; + + // just for refreshing UIs when bone space was changed + static TSharedPtr PoseAIHandTargetDelegate; + +public: + // UObject interface + virtual void Serialize(FArchive& Ar) override; + // End of UObject interface + + // UEdGraphNode interface + virtual FText GetNodeTitle(ENodeTitleType::Type TitleType) const override; + virtual FText GetTooltipText() const override; + // End of UEdGraphNode interface + + // UAnimGraphNode_Base interface + virtual void CustomizeDetails(class IDetailLayoutBuilder& DetailBuilder) override; + //virtual FEditorModeID GetEditorMode() const; + virtual void CopyNodeDataToPreviewNode(FAnimNode_Base* InPreviewNode) override; + virtual void CopyPinDefaultsToNodeData(UEdGraphPin* InPin) override; + // End of UAnimGraphNode_Base interface + + // UAnimGraphNode_SkeletalControlBase interface + virtual const FAnimNode_SkeletalControlBase* GetNode() const override { return &Node; } + // End of UAnimGraphNode_SkeletalControlBase interface + + IDetailLayoutBuilder* DetailLayout; + +protected: + // UAnimGraphNode_SkeletalControlBase interface + virtual void Draw(FPrimitiveDrawInterface* PDI, USkeletalMeshComponent* SkelMeshComp) const override; + virtual FText GetControllerDescription() const override; + // End of UAnimGraphNode_SkeletalControlBase interface + +private: + /** Constructing FText strings can be costly, so we cache the node's title */ + FNodeTitleTextTable CachedNodeTitles; +}; diff --git a/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h new file mode 100644 index 0000000..c0b4d2e --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/5.4/PoseAILiveLink/Source/PoseAILiveLinkEd/Public/PoseAILiveLinkEd.h @@ -0,0 +1,21 @@ +// Copyright Pose AI Ltd 2022. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Modules/ModuleManager.h" + + +class FPoseAILiveLinkEdModule : public IModuleInterface +{ +public: + + /** IModuleInterface implementation */ + virtual void StartupModule() override; + virtual void ShutdownModule() override; + +private: + +}; + + diff --git a/UnrealEngineAPI/PluginV3.0/VersionNotes.txt b/UnrealEngineAPI/PluginV3.0/VersionNotes.txt new file mode 100644 index 0000000..f623df5 --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/VersionNotes.txt @@ -0,0 +1,13 @@ +Keeping a note here of the main differences between UE5 vs UE4 version builds for quick conversion. +TBD: Can we do this with compiler defines that the build system recognizes and avoid having 4 versions of the plugin? + +PoseAiMediaFramework.cpp +The UPoseAIMediaComponent::TickComponent method generates a FMediaTextureResource. In UE4 this was MediaTexture->Resource, in UE5 MediaTexture->GetResource() + +PoseAIUdpSocketReceiver.h +In Update method, in 4.0 we use UTF8_TO_TChar to make the FString (not as safe as desired, has been crash prone in past), in 5.0 we use a newer (and safer) FString constructer. + +PoseAILiveLinkBuild.cs +UE5 requires LiveLinkAnimationCore in PublicDependencies. Inclusion is automatically handled with code. + + diff --git a/UnrealEngineAPI/PluginV3.0/readme.md b/UnrealEngineAPI/PluginV3.0/readme.md new file mode 100644 index 0000000..755152f --- /dev/null +++ b/UnrealEngineAPI/PluginV3.0/readme.md @@ -0,0 +1,37 @@ +# Revised version 1.4 plugin for Unreal +This version includes a significant refactoring of the LiveLink source files. +We believe these changes should create a more stable, easier-to-maintain plugin with safer memory management. + +We plan to refactor the network code and json processing in the future. + +## "Breaking" change - Enums in Handshake +We have replaced the human-error prone string fields in our handshakes with enums. +This will likely cause existing blueprints to show an error on compile. + +Right click on the node and select "refresh node" and the errors should clear. Check the mode and rig fields in particular as they will be reset to the defaults post update! + +Apologies for the disruption but this is better for most users in the medium to long term. + +## New IK node +We have added a new animation blueprint node for better arm placement on arbitrary avatars, without requiring calibration with the real life user. +We are adding them to the demo project. + +To set the node up yourself, you need to specify the four joints (left/right upper arms, lowest spine joint and the wrist joint) from your rig and then plug in the left or right hand ik vector from our stream (with the supplementary data in live values). + +See our video on youtube or our updated example for a functional setup. + + +## Sync FPS setting +Sync Fps sets the on-device engine to interpolate and stream at a steady rate, potentially different then the speed of the camera or erratic mobile OS processing might provide. This comes at a cost: buffering of around 0.5-1.0 camera frames which introduces 15-30ms of motion capture lag. + +Setting the Sync FPS to zero should disable the interpolation, although this setting was not working as intended in app/engine versions prior to 1.3. + +We would currently recommend setting Sync Fps to 0 on newer versions of the engine, to minimize lag. You can instead use Sync Fps if you find the camera device too erratic. + + +## Integrated camera plugin +This version also adds functionality needed for our supplementary Windows media player plugin to use our engine in Window and in-engine with a webcam or video player. + +Note: the integrated camera supplementary plugin and functionality is not on general release as of yet. + + diff --git a/demo_python.py b/demo_python.py index 79d1ab1..e15d14f 100644 --- a/demo_python.py +++ b/demo_python.py @@ -38,6 +38,9 @@ def show_my_ip(): 'mirror': 'YES', # Options: 'YES', 'NO' 'syncFPS': 60, # App smooths processed frames to constant FPS. 0 for async mode (will still smooth joints between processed frames) 'cameraFPS': 60, # Tries to set camera to this speed. 30 and 60 work on most iphone cameras + 'packetFormat': 0, # 0 if verbose and human readable, but may exceed packet size limits. 1 is compact and what we use normally. + 'signature' : '', + 'whoami': '' } } ).encode()