// Copyright 2021 Pose AI Ltd. All rights reserved using UnityEngine; using System.Collections.Generic; using System.Reflection; using System.Diagnostics; using System.Linq; using System; namespace PoseAI { public static class PoseAIRigFactory { public static PoseAIRigBase SelectRig(PoseAI_Rigs rigType) => rigType switch { PoseAI_Rigs.Unity => new PoseAIRigUnity(), PoseAI_Rigs.UE4 => new PoseAIRigUE4(), PoseAI_Rigs.Mixamo => new PoseAIRigMixamo(), _ => new PoseAIRigUnity() }; } [System.Serializable] public class PoseAIHandshake { [System.Serializable] public class Handshake { //name displayed in the Pose Camera app upon connection public string name = "Unity Demo"; public string rig; public string mode; public string mirror; public int syncFPS; public int cameraFPS; // name of app and signature for licensee connections public string whoami = "Your App Name"; public string signature = "Your App Signature"; // 0 for verbose JSON, 1 for more compact packets. public int packetFormat = 1; //this is the version of our AI. version 1 is our original model during beta and release. version 2 was released in March 2022 and is currently the default. public int modelVersion = 2; } public Handshake HANDSHAKE = new Handshake(); [System.Serializable] public class Config { // sensitivities for the event triggers. 0.0 is lowest sensitivity (more false negatives), 1.0 is highest (more false positivies) public float stepSensitivity = 0.75f; public float jumpSensitivity = 0.25f; public float armSensitivity = 0.5f; public float crouchSensitivity = 0.5f; } public Config CONFIG = new Config(); //public double echoServerTimestamp = ; public static PoseAIHandshake Factory(PoseAI_Modes mode, PoseAI_Rigs rig, bool mirror, int syncFPS, int cameraFPS) { PoseAIHandshake handshake = new PoseAIHandshake(); handshake.HANDSHAKE.rig = rig.ToString(); handshake.HANDSHAKE.mode = mode.ToString(); handshake.HANDSHAKE.mirror = mirror ? "YES" : "NO"; handshake.HANDSHAKE.cameraFPS = Mathf.Max(cameraFPS, PoseAIConfig.MIN_CAMERA_FPS); handshake.HANDSHAKE.syncFPS = Mathf.Max(handshake.HANDSHAKE.cameraFPS, syncFPS); return handshake; } } [System.Serializable] public abstract class PoseAIRigBase { // possible fields from incoming JSON stream public string userName; public string version; public string deviceName; public string sessionUUID; //this is cleared and stored privately to help track if a new handshake was sent private string sessionUUID_; public int ModelLatency = 0; public double Timestamp; // skeleton joint naming and base rotation, such as UE4, Mixamo etc. public string Rig; // packet format. 0 is verbose, 1 is compressed public int PF; //receives compact touch updates from app, and adds them as vector 2s to Queue for game logic processing. Values are relative to phone not user (use orientation to rotate as appropriate) public string TouchState = ""; public string Touches = ""; public Queue touchQueue = new Queue(); public List touchStates = UITouch.MakeList(10); //if touches are enabled, sends current device orientation. For iOS this is the UIDeviceOrientation enum public int Orientation; public VisibilityFlags visibility = new VisibilityFlags(); // extra rotation applied to hip/pelvis. public Quaternion baseRotation; // whether app is in desktop mode (Excluding legs) public bool isDesktop = false; //number of joints to skip in desktop mode public int lowerBodyNumOfJoints = 8; // this is the order the PoseCamera sends the joints. public static readonly List bones = new List{ HumanBodyBones.Hips, HumanBodyBones.RightUpperLeg, HumanBodyBones.RightLowerLeg, HumanBodyBones.RightFoot, HumanBodyBones.RightToes, HumanBodyBones.LeftUpperLeg, HumanBodyBones.LeftLowerLeg, HumanBodyBones.LeftFoot, HumanBodyBones.LeftToes, HumanBodyBones.Spine, HumanBodyBones.Chest, HumanBodyBones.UpperChest, HumanBodyBones.Neck, HumanBodyBones.Head, HumanBodyBones.LeftShoulder, HumanBodyBones.LeftUpperArm, HumanBodyBones.LeftLowerArm, HumanBodyBones.RightShoulder, HumanBodyBones.RightUpperArm, HumanBodyBones.RightLowerArm, HumanBodyBones.LeftHand, HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value HumanBodyBones.LeftIndexProximal, HumanBodyBones.LeftIndexIntermediate, HumanBodyBones.LeftIndexDistal, HumanBodyBones.LeftMiddleProximal, HumanBodyBones.LeftMiddleIntermediate, HumanBodyBones.LeftMiddleDistal, HumanBodyBones.LeftRingProximal, HumanBodyBones.LeftRingIntermediate, HumanBodyBones.LeftRingDistal, HumanBodyBones.LeftLittleProximal, HumanBodyBones.LeftLittleIntermediate, HumanBodyBones.LeftLittleDistal, HumanBodyBones.LeftThumbProximal, HumanBodyBones.LeftThumbIntermediate, HumanBodyBones.LeftThumbDistal, HumanBodyBones.RightHand, HumanBodyBones.LastBone, //forearm twist not in unity avatar system, using LastBone as a Null value HumanBodyBones.RightIndexProximal, HumanBodyBones.RightIndexIntermediate, HumanBodyBones.RightIndexDistal, HumanBodyBones.RightMiddleProximal, HumanBodyBones.RightMiddleIntermediate, HumanBodyBones.RightMiddleDistal, HumanBodyBones.RightRingProximal, HumanBodyBones.RightRingIntermediate, HumanBodyBones.RightRingDistal, HumanBodyBones.RightLittleProximal, HumanBodyBones.RightLittleIntermediate, HumanBodyBones.RightLittleDistal, HumanBodyBones.RightThumbProximal, HumanBodyBones.RightThumbIntermediate, HumanBodyBones.RightThumbDistal }; public static readonly List parentIndices = new List { 0,0,1,2,3,0,5,6,7,0,9,10,11,12,11,14,15,11,17,18,16,0,20,22,23,20,25,26,20,28,29,20,31,32,20,34,35,19,0,37,39,40,37,42,43,37,45,46,37,48,49,37,51,52, }; protected internal List> rotationData; protected internal List rotationNames; protected internal int numOfLeftHandJoints; protected internal int numOfRightHandJoints; protected internal int numOfBodyJoints; private Stopwatch stopwatch = Stopwatch.StartNew(); private bool newHandshake = false; // baseRotationIn: extra rotation applied to hip/pelvis. This will depend on how the rig was setup in modeling package and how it was imported from FBX. You may need to change the values used in the specific rig subclasses. public PoseAIRigBase(Quaternion baseRotationIn) { baseRotation = baseRotationIn; baseRotation.Normalize(); rotationData = new List>(); rotationNames = new List(); BuildListsFromIntrospection(); } public bool OverwriteFromJSON(string jsonString) { // if the port is exposed as a public server, expect to get lots of false connections (i.e. port sniffers or malicious actors). unity doesn't have a json verifier so will use a try/catch. try { JsonUtility.FromJsonOverwrite(jsonString, this); } catch (Exception ex) { return false; } if (!string.IsNullOrEmpty(sessionUUID)) { sessionUUID_ = sessionUUID; sessionUUID = null; newHandshake = true; } else { newHandshake = false; stopwatch.Restart(); } if (PF == 1) { GetBody().Scalars.ProcessCompact(ref GetBody().ScaA); GetBody().Vectors.ProcessCompact(ref GetBody().VecA); GetBody().Vectors.ProcessCompact(ref GetBody().VecA); GetBody().Events.ProcessCompact(ref GetBody().EveA); visibility.ProcessCompact(ref GetBody().VisA); } else { visibility.ProcessVerbose(ref GetBody().Scalars); } ProcessTouches(); return true; } public Quaternion GetBaseRotation() { return baseRotation; } public bool HasRigInfo() { return !string.IsNullOrEmpty(Rig) && (PF == 1) ? (GetBody().RotA.Length > 7 ) : (rotationData.Count > 0 && rotationData[0].Count == 4); } public bool IsIncomingHandshake() { return newHandshake && !string.IsNullOrEmpty(version); } public bool IsStale() { return !HasRigInfo() || stopwatch.ElapsedMilliseconds >= PoseAIConfig.STALE_TIME_IN_MS; } public virtual BaseBodyWrapper GetBody() { return null; } public PoseAIEvents GetEvents() { return GetBody().Events; } public List GetJointNames() { return rotationNames; } public virtual BaseHandWrapper GetLeftHand() { return null; } public virtual BaseHandWrapper GetRightHand() { return null; } public string GetRootName() { return rotationNames.Count > 0 ? rotationNames[0] : null; } public Tuple, List> GetRotationsCameraSpace() { List validity = new List(rotationNames.Count); List quats = new List(rotationNames.Count); if (PF == 1) { AppendCompactRotations(ref GetBody().RotA, ref quats); validity.AddRange(Enumerable.Repeat(true, quats.Count)); if (isDesktop && validity.Count >= 1 + lowerBodyNumOfJoints) { for(int i = 1; i< lowerBodyNumOfJoints + 1; ++i) validity[i] = false; } if (GetLeftHand().RotA.Length / 8 != numOfLeftHandJoints) { quats.AddRange(Enumerable.Repeat(Quaternion.identity, numOfLeftHandJoints)); validity.AddRange(Enumerable.Repeat(false, numOfLeftHandJoints)); } else { AppendCompactRotations(ref GetLeftHand().RotA, ref quats); validity.AddRange(Enumerable.Repeat(true, numOfLeftHandJoints)); } if (GetRightHand().RotA.Length / 8 != numOfRightHandJoints) { quats.AddRange(Enumerable.Repeat(Quaternion.identity, numOfRightHandJoints)); validity.AddRange(Enumerable.Repeat(false, numOfRightHandJoints)); } else { AppendCompactRotations(ref GetRightHand().RotA, ref quats); validity.AddRange(Enumerable.Repeat(true, numOfRightHandJoints)); } } else { foreach (List data in rotationData) { Quaternion quat = new Quaternion(); validity.Add(QuatFromList(data, ref quat)); quats.Add(quat); } } return new Tuple, List>(validity, quats); } private void AppendCompactRotations(ref string compactString, ref List quats) { if (compactString.Length > 7) { List flatArray = new List(compactString.Length / 2); PoseAI_Decoder.FStringFixed12ToFloat(ref compactString, ref flatArray); PoseAI_Decoder.FlatArrayToQuats(ref flatArray, ref quats); } } private void ProcessTouches() { if (Touches.Length > 1) { // safety valve which clears the queue at a certain size, in case the program is not processing touches if (touchQueue.Count > PoseAIConfig.MAX_SIZE_TOUCH_QUEUE) touchQueue.Clear(); UITouch.DecodeMultiTouchString(ref Touches, ref touchQueue); Touches = ""; } if (TouchState.Length > 0) UITouch.DecodeMultiTouchStateString(ref TouchState, ref touchStates); } // convenience routine to get rotation field names and pointers into lists. // Only run at creation to avoid introspection overhead, as rig object is reused each frame protected internal virtual void BuildListsFromIntrospection() { } // converts float list to quaternion and normalizes, adjusting for differences in Unity coordinate system // returns true or false if a valid quat is formed (since null is not accepted for quats) private static bool QuatFromList(List quatAsList, ref Quaternion quat) { if (quatAsList == null || quatAsList.Count != 4) { return false; } else { quat.Set(quatAsList[1], -quatAsList[0], quatAsList[3], -quatAsList[2]); quat.Normalize(); return true; } } } [System.Serializable] public abstract class BaseBodyWrapper { public VectorsBody Vectors = new VectorsBody(); public ScalarsBody Scalars = new ScalarsBody(); public PoseAIEvents Events = new PoseAIEvents(); // compressed format fields public string VisA = ""; public string ScaA = ""; public string EveA = ""; public string VecA = ""; public string RotA = ""; public virtual BaseRotationsWrapper GetRotations() { return null; } } [System.Serializable] public abstract class BaseRotationsWrapper { // introspection helper function public void AddFields(List> rotationData, List rotationNames) { foreach (var field in this.GetType().GetFields()) { field.SetValue(this, new List()); rotationData.Add(field.GetValue(this) as List); rotationNames.Add(field.Name); } } public int NumOfJoints() { return this.GetType().GetFields().Length; } } [System.Serializable] public class VectorsBody { public List HipScreen = new List { 0.0f, 0.0f }; public List ChestScreen = new List { 0.0f, 0.0f }; public List HipLean = new List { 0.0f, 0.0f }; public void ProcessCompact(ref string compactString) { if (compactString.Length < 12) return; HipLean[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[0], compactString[1]) * 180.0f; HipLean[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f; HipScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[4], compactString[5]); HipScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[6], compactString[7]); ChestScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[8], compactString[9]); ChestScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[10], compactString[11]); } } [System.Serializable] public class ScalarsBody { /** visibility flags, corresponding to figure in the app. Use the visibility class in the Rig class instead as these do not handle compact packets or track updates **/ public float VisTorso = 0.0f; public float VisArmL = 0.0f; public float VisArmR = 0.0f; public float VisLegL = 0.0f; public float VisLegR = 0.0f; /** location of left hand relative to body in broad zones */ public uint HandZoneL = 5; private uint InternalHandZoneL = 5; /** location of right hand relative to body in broad zones */ public uint HandZoneR = 5; private uint InternalHandZoneR = 5; /** Heading in degrees of torso. 0 is heading to camera */ public float ChestYaw = 0.0f; /** Heading in degrees of flattened left foot to right foot vector relative to camera. 0 is parallel to camera */ public float StanceYaw = 0.0f; /** estimated actual height of the subject in clip coordinates (2.0 = full height of image) */ public float BodyHeight = 0.0f; /** whether subject is crouching */ public uint IsCrouching = 0; private uint InternalIsCrouching = 0; /** number of feet that have been stable for a few frames */ public uint StableFoot = 0; private uint InternalStableFoot = 0; public bool CrouchHasChangedAndUpdate(){return HasChangedAndUpdate(ref InternalIsCrouching, IsCrouching);} public bool StableFootHasChangedAndUpdate() { return HasChangedAndUpdate(ref InternalStableFoot, StableFoot); } public bool HandZoneLHasChangedAndUpdate() { return HasChangedAndUpdate(ref InternalHandZoneL, HandZoneL); } public bool HandZoneRHasChangedAndUpdate() { return HasChangedAndUpdate(ref InternalHandZoneR, HandZoneR); } private bool HasChangedAndUpdate(ref uint InternalValue, uint value) { bool hasTriggered = InternalValue != value; InternalValue = value; return hasTriggered; } public void ProcessCompact(ref string compactString){ if (compactString.Length < 14) return; BodyHeight = PoseAI_Decoder.FixedB64pairToFloat(compactString[0], compactString[1]) + 1.0f; ChestYaw = PoseAI_Decoder.FixedB64pairToFloat(compactString[2], compactString[3]) * 180.0f; StanceYaw = PoseAI_Decoder.FixedB64pairToFloat(compactString[4], compactString[5]) * 180.0f; StableFoot = PoseAI_Decoder.UintB64ToUint(compactString[6], compactString[7]); HandZoneL = PoseAI_Decoder.UintB64ToUint(compactString[8], compactString[9]); HandZoneR = PoseAI_Decoder.UintB64ToUint(compactString[10], compactString[11]); IsCrouching = PoseAI_Decoder.UintB64ToUint(compactString[12], compactString[13]); } } public class VisibilityFlags { public bool isTorso = false; public bool isLeftArm = false; public bool isRightArm = false; public bool isLeftLeg = false; public bool isRightLeg = false; public bool HasChanged() { return hasChanged; } public void ProcessVerbose(ref ScalarsBody bodyVerbose) { hasChanged = false; SetAndCheckForChange(bodyVerbose.VisTorso > 0.5f, ref isTorso, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisLegL > 0.5f, ref isLeftLeg, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisLegR > 0.5f, ref isRightLeg, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisArmL > 0.5f, ref isLeftArm, ref hasChanged); SetAndCheckForChange(bodyVerbose.VisArmR > 0.5f, ref isRightArm, ref hasChanged); } public void ProcessCompact(ref string visString) { hasChanged = false; SetAndCheckForChange(visString[0] != '0', ref isTorso, ref hasChanged); SetAndCheckForChange(visString[1] != '0', ref isLeftLeg, ref hasChanged); SetAndCheckForChange(visString[2] != '0', ref isRightLeg, ref hasChanged); SetAndCheckForChange(visString[3] != '0', ref isLeftArm, ref hasChanged); SetAndCheckForChange(visString[4] != '0', ref isRightArm, ref hasChanged); } private bool hasChanged = false; private void SetAndCheckForChange(bool newValue, ref bool field, ref bool changeFlag) { if (newValue != field) changeFlag = true; field = newValue; } } [System.Serializable] public class EventPairBase { /** number of events registered by camera */ public uint Count = 0; /* once per update use this function to check each relevant event to see if a trigger was detected. It will update the internal counter */ public bool CheckTriggerAndUpdate() { bool hasTriggered = InternalCount != Count; InternalCount = Count; return hasTriggered; } private uint InternalCount = 0; public virtual void ProcessCompact(string compactString) { } } [System.Serializable] public class EventPair : EventPairBase { /**magnitude of event where approrpiate */ public float Magnitude = 0.0f; public override void ProcessCompact(string compactString) { Count = PoseAI_Decoder.UintB64ToUint(compactString[0], compactString[1], compactString[2]); Magnitude = PoseAI_Decoder.FixedB64pairToFloat(compactString[3], compactString[4]); } } [System.Serializable] public class GesturePair : EventPairBase { /**index code for most recent gesture */ public uint Current = 0; public override void ProcessCompact(string compactString) { Count = PoseAI_Decoder.UintB64ToUint(compactString[0], compactString[1], compactString[2]); Current = PoseAI_Decoder.UintB64ToUint(compactString[3], compactString[4]); } public PoseAI_Gestures GetGesture() { return PoseAI_Methods.Gesture(Current); } } /* structure to receive event notifications */ [System.Serializable] public class PoseAIEvents { /** number of footsteps registered by camera, body height magnitude */ public EventPair Footstep = new EventPair(); /** number of left foot sidesteps registered by camera. sign indicates direction */ public EventPair SidestepL = new EventPair(); /** number of right foot sidesteps registered by camera. sign indicates direction */ public EventPair SidestepR = new EventPair(); /** number of jumps registered by camera */ public EventPair Jump = new EventPair(); /** number of footsplits registered by camera, body width magnitude */ public EventPair FeetSplit = new EventPair(); /** number of arm pumps registered by camera, body height magnitude */ public EventPair ArmPump = new EventPair(); /** number of arm flexes registered by camera, body width magnitude */ public EventPair ArmFlex = new EventPair(); /** number of left or dual arm gestures registered by camera and most recent gesture */ public GesturePair ArmGestureL = new GesturePair(); /** number of right arm gestures registered by camera and most recent gesture */ public GesturePair ArmGestureR = new GesturePair(); public PoseAIEvents() { compactOrder = new List { Footstep, SidestepL, SidestepR, Jump, FeetSplit, ArmPump, ArmFlex, ArmGestureL, ArmGestureR }; } public void ClearEventTriggers() { foreach(var e in compactOrder) e.CheckTriggerAndUpdate(); } public void ProcessCompact(ref string compactString) { List compactOrder = new List { Footstep, SidestepL, SidestepR, Jump, FeetSplit, ArmPump, ArmFlex, ArmGestureL, ArmGestureR }; if (compactString.Length % 5 != 0) { UnityEngine.Debug.Log("PoseAI: Invalid event string."); return; } for (int i = 0; i < compactOrder.Count; ++i) { if (compactString.Length < 5 * i) break; compactOrder[i].ProcessCompact(compactString.Substring(i * 5, 5)); } } private List compactOrder; } [System.Serializable] public abstract class BaseHandWrapper { public VectorsHand Vectors = new VectorsHand(); public ScalarsHand Scalars = new ScalarsHand(); // compressed format field public string RotA; } [System.Serializable] public class ScalarsHand { } [System.Serializable] public class VectorsHand { public List PointScreen = new List { 0.0f, 0.0f }; public void ProcessCompact(ref string compactString) { if (compactString.Length < 4) return; PointScreen[0] = PoseAI_Decoder.FixedB64pairToFloat(compactString[0], compactString[1]); PointScreen[1] = PoseAI_Decoder.FixedB64pairToFloat(compactString[2], compactString[3]); } } }