using System;
using System.Collections.Generic;
using UnityEngine;
namespace Unity.MLAgents.Sensors
{
///
/// Determines which dimensions the sensor will perform the casts in.
///
public enum RayPerceptionCastType
{
///
/// Cast in 2 dimensions, using Physics2D.CircleCast or Physics2D.RayCast.
///
Cast2D,
///
/// Cast in 3 dimensions, using Physics.SphereCast or Physics.RayCast.
///
Cast3D,
}
///
/// Contains the elements that define a ray perception sensor.
///
public struct RayPerceptionInput
{
///
/// Length of the rays to cast. This will be scaled up or down based on the scale of the transform.
///
public float RayLength;
///
/// List of tags which correspond to object types agent can see.
///
public IReadOnlyList DetectableTags;
///
/// List of angles (in degrees) used to define the rays.
/// 90 degrees is considered "forward" relative to the game object.
///
public IReadOnlyList Angles;
///
/// Starting height offset of ray from center of agent
///
public float StartOffset;
///
/// Ending height offset of ray from center of agent.
///
public float EndOffset;
///
/// Radius of the sphere to use for spherecasting.
/// If 0 or less, rays are used instead - this may be faster, especially for complex environments.
///
public float CastRadius;
///
/// Transform of the GameObject.
///
public Transform Transform;
///
/// Whether to perform the casts in 2D or 3D.
///
public RayPerceptionCastType CastType;
///
/// Filtering options for the casts.
///
public int LayerMask;
///
/// Returns the expected number of floats in the output.
///
///
public int OutputSize()
{
return OutputSizePerRay() * NumRays();
}
public int OutputSizePerRay()
{
return (DetectableTags?.Count ?? 0) + 2;
}
public int NumRays()
{
return Angles?.Count ?? 0;
}
///
/// Get the cast start and end points for the given ray index/
///
///
/// A tuple of the start and end positions in world space.
public (Vector3 StartPositionWorld, Vector3 EndPositionWorld) RayExtents(int rayIndex)
{
var angle = Angles[rayIndex];
Vector3 startPositionLocal, endPositionLocal;
if (CastType == RayPerceptionCastType.Cast3D)
{
startPositionLocal = new Vector3(0, StartOffset, 0);
endPositionLocal = PolarToCartesian3D(RayLength, angle);
endPositionLocal.y += EndOffset;
}
else
{
// Vector2s here get converted to Vector3s (and back to Vector2s for casting)
startPositionLocal = new Vector2();
endPositionLocal = PolarToCartesian2D(RayLength, angle);
}
var startPositionWorld = Transform.TransformPoint(startPositionLocal);
var endPositionWorld = Transform.TransformPoint(endPositionLocal);
return (StartPositionWorld: startPositionWorld, EndPositionWorld: endPositionWorld);
}
///
/// Converts polar coordinate to cartesian coordinate.
///
static internal Vector3 PolarToCartesian3D(float radius, float angleDegrees)
{
var x = radius * Mathf.Cos(Mathf.Deg2Rad * angleDegrees);
var z = radius * Mathf.Sin(Mathf.Deg2Rad * angleDegrees);
return new Vector3(x, 0f, z);
}
///
/// Converts polar coordinate to cartesian coordinate.
///
static internal Vector2 PolarToCartesian2D(float radius, float angleDegrees)
{
var x = radius * Mathf.Cos(Mathf.Deg2Rad * angleDegrees);
var y = radius * Mathf.Sin(Mathf.Deg2Rad * angleDegrees);
return new Vector2(x, y);
}
}
///
/// Contains the data generated/produced from a ray perception sensor.
///
public class RayPerceptionOutput
{
///
/// Contains the data generated from a single ray of a ray perception sensor.
///
public struct RayOutput
{
///
/// Whether or not the ray hit anything.
///
public bool HasHit;
///
/// Whether or not the ray hit an object whose tag is in the input's DetectableTags list.
///
public bool HitTaggedObject;
///
/// The index of the hit object's tag in the DetectableTags list, or -1 if there was no hit, or the
/// hit object has a different tag.
///
public int HitTagIndex;
///
/// Normalized distance to the hit object.
///
public float HitFraction;
///
/// The hit GameObject (or null if there was no hit).
///
public GameObject HitGameObject;
///
/// Start position of the ray in world space.
///
public Vector3 StartPositionWorld;
///
/// End position of the ray in world space.
///
public Vector3 EndPositionWorld;
///
/// The scaled length of the ray.
///
///
/// If there is non-(1,1,1) scale, |EndPositionWorld - StartPositionWorld| will be different from
/// the input rayLength.
///
public float ScaledRayLength
{
get
{
var rayDirection = EndPositionWorld - StartPositionWorld;
return rayDirection.magnitude;
}
}
///
/// The scaled size of the cast.
///
///
/// If there is non-(1,1,1) scale, the cast radius will be also be scaled.
///
public float ScaledCastRadius;
///
/// Writes the ray output information to a subset of the float array. Each element in the rayAngles array
/// determines a sublist of data to the observation. The sublist contains the observation data for a single cast.
/// The list is composed of the following:
/// 1. A one-hot encoding for detectable tags. For example, if DetectableTags.Length = n, the
/// first n elements of the sublist will be a one-hot encoding of the detectableTag that was hit, or
/// all zeroes otherwise.
/// 2. The 'numDetectableTags' element of the sublist will be 1 if the ray missed everything, or 0 if it hit
/// something (detectable or not).
/// 3. The 'numDetectableTags+1' element of the sublist will contain the normalized distance to the object
/// hit, or 1.0 if nothing was hit.
///
///
///
/// Output buffer. The size must be equal to (numDetectableTags+2) * RayOutputs.Length
public void ToFloatArray(int numDetectableTags, int rayIndex, float[] buffer)
{
var bufferOffset = (numDetectableTags + 2) * rayIndex;
if (HitTaggedObject)
{
buffer[bufferOffset + HitTagIndex] = 1f;
}
buffer[bufferOffset + numDetectableTags] = HasHit ? 0f : 1f;
buffer[bufferOffset + numDetectableTags + 1] = HitFraction;
}
}
public int CustomObservationSizePerRay;
public virtual void GetCustomObservationData(RayOutput rayOutput, float[] buffer) { }
///
/// RayOutput for each ray that was cast.
///
public RayOutput[] RayOutputs;
}
///
/// A sensor implementation that supports ray cast-based observations.
///
public class RayPerceptionSensor : ISensor, IBuiltInSensor
{
float[] m_Observations;
float[] m_SingleRayObservation;
ObservationSpec m_ObservationSpec;
string m_Name;
RayPerceptionInput m_RayPerceptionInput;
RayPerceptionOutput m_RayPerceptionOutput;
///
/// Time.frameCount at the last time Update() was called. This is only used for display in gizmos.
///
int m_DebugLastFrameCount;
internal int DebugLastFrameCount
{
get { return m_DebugLastFrameCount; }
}
internal int ObservationSizePerRay
{
get { return m_RayPerceptionOutput.CustomObservationSizePerRay + ((m_RayPerceptionInput.DetectableTags?.Count ?? 0) + 2); }
}
///
/// Creates the RayPerceptionSensor.
///
/// The name of the sensor.
/// The inputs for the sensor.
public RayPerceptionSensor(string name, RayPerceptionInput rayInput)
{
m_Name = name;
m_RayPerceptionInput = rayInput;
SetNumObservations(rayInput.OutputSize());
m_DebugLastFrameCount = Time.frameCount;
m_RayPerceptionOutput = new RayPerceptionOutput();
}
///
/// Creates the RayPerceptionSensor.
///
/// The name of the sensor.
/// The inputs for the sensor.
/// The outputs for the sensor.
public RayPerceptionSensor(string name, RayPerceptionInput rayInput, RayPerceptionOutput rayOutput)
{
m_Name = name;
m_RayPerceptionInput = rayInput;
SetNumObservations((rayOutput.CustomObservationSizePerRay + rayInput.OutputSizePerRay()) * rayInput.NumRays());
if (rayOutput.CustomObservationSizePerRay > 0)
{
m_SingleRayObservation = new float[rayOutput.CustomObservationSizePerRay];
}
m_DebugLastFrameCount = Time.frameCount;
m_RayPerceptionOutput = rayOutput;
}
///
/// The most recent raycast results.
///
public RayPerceptionOutput RayPerceptionOutput
{
get { return m_RayPerceptionOutput; }
}
void SetNumObservations(int numObservations)
{
m_ObservationSpec = ObservationSpec.Vector(numObservations);
m_Observations = new float[numObservations];
}
internal void SetRayPerceptionInput(RayPerceptionInput rayInput)
{
// Note that change the number of rays or tags doesn't directly call this,
// but changing them and then changing another field will.
if (m_RayPerceptionInput.OutputSize() != rayInput.OutputSize())
{
Debug.Log(
"Changing the number of tags or rays at runtime is not " +
"supported and may cause errors in training or inference."
);
// Changing the shape will probably break things downstream, but we can at least
// keep this consistent.
SetNumObservations(rayInput.OutputSize());
}
m_RayPerceptionInput = rayInput;
}
///
/// Computes the ray perception observations and saves them to the provided
/// .
///
/// Where the ray perception observations are written to.
///
public int Write(ObservationWriter writer)
{
using (TimerStack.Instance.Scoped("RayPerceptionSensor.Perceive"))
{
Array.Clear(m_Observations, 0, m_Observations.Length);
var numRays = m_RayPerceptionInput.Angles.Count;
var numDetectableTags = m_RayPerceptionInput.DetectableTags.Count;
// For each ray, write the information to the observation buffer
for (var rayIndex = 0; rayIndex < numRays; rayIndex++)
{
m_RayPerceptionOutput.RayOutputs?[rayIndex].ToFloatArray(numDetectableTags, rayIndex, m_Observations);
if (m_RayPerceptionOutput.CustomObservationSizePerRay > 0)
{
Array.Clear(m_SingleRayObservation, 0, m_SingleRayObservation.Length);
m_RayPerceptionOutput.GetCustomObservationData(m_RayPerceptionOutput.RayOutputs[rayIndex], m_SingleRayObservation);
Array.Copy(
m_SingleRayObservation,
0,
m_Observations,
ObservationSizePerRay * rayIndex + m_RayPerceptionInput.OutputSizePerRay(),
m_RayPerceptionOutput.CustomObservationSizePerRay
);
}
}
// Finally, add the observations to the ObservationWriter
writer.AddList(m_Observations);
}
return m_Observations.Length;
}
///
public void Update()
{
m_DebugLastFrameCount = Time.frameCount;
var numRays = m_RayPerceptionInput.Angles.Count;
if (m_RayPerceptionOutput.RayOutputs == null || m_RayPerceptionOutput.RayOutputs.Length != numRays)
{
m_RayPerceptionOutput.RayOutputs = new RayPerceptionOutput.RayOutput[numRays];
}
// For each ray, do the casting and save the results.
for (var rayIndex = 0; rayIndex < numRays; rayIndex++)
{
m_RayPerceptionOutput.RayOutputs[rayIndex] = PerceiveSingleRay(m_RayPerceptionInput, rayIndex);
}
}
///
public void Reset() { }
///
public ObservationSpec GetObservationSpec()
{
return m_ObservationSpec;
}
///
public string GetName()
{
return m_Name;
}
///
public virtual byte[] GetCompressedObservation()
{
return null;
}
///
public CompressionSpec GetCompressionSpec()
{
return CompressionSpec.Default();
}
///
public BuiltInSensorType GetBuiltInSensorType()
{
return BuiltInSensorType.RayPerceptionSensor;
}
///
/// Evaluates the raycasts to be used as part of an observation of an agent.
///
/// Input defining the rays that will be cast.
/// Output struct containing the raycast results.
public static RayPerceptionOutput Perceive(RayPerceptionInput input)
{
RayPerceptionOutput output = new RayPerceptionOutput();
output.RayOutputs = new RayPerceptionOutput.RayOutput[input.Angles.Count];
for (var rayIndex = 0; rayIndex < input.Angles.Count; rayIndex++)
{
output.RayOutputs[rayIndex] = PerceiveSingleRay(input, rayIndex);
}
return output;
}
///
/// Evaluate the raycast results of a single ray from the RayPerceptionInput.
///
///
///
///
internal static RayPerceptionOutput.RayOutput PerceiveSingleRay(
RayPerceptionInput input,
int rayIndex
)
{
var unscaledRayLength = input.RayLength;
var unscaledCastRadius = input.CastRadius;
var extents = input.RayExtents(rayIndex);
var startPositionWorld = extents.StartPositionWorld;
var endPositionWorld = extents.EndPositionWorld;
var rayDirection = endPositionWorld - startPositionWorld;
// If there is non-unity scale, |rayDirection| will be different from rayLength.
// We want to use this transformed ray length for determining cast length, hit fraction etc.
// We also it to scale up or down the sphere or circle radii
var scaledRayLength = rayDirection.magnitude;
// Avoid 0/0 if unscaledRayLength is 0
var scaledCastRadius = unscaledRayLength > 0 ?
unscaledCastRadius * scaledRayLength / unscaledRayLength :
unscaledCastRadius;
// Do the cast and assign the hit information for each detectable tag.
var castHit = false;
var hitFraction = 1.0f;
GameObject hitObject = null;
if (input.CastType == RayPerceptionCastType.Cast3D)
{
#if MLA_UNITY_PHYSICS_MODULE
RaycastHit rayHit;
if (scaledCastRadius > 0f)
{
castHit = Physics.SphereCast(startPositionWorld, scaledCastRadius, rayDirection, out rayHit,
scaledRayLength, input.LayerMask);
}
else
{
castHit = Physics.Raycast(startPositionWorld, rayDirection, out rayHit,
scaledRayLength, input.LayerMask);
}
// If scaledRayLength is 0, we still could have a hit with sphere casts (maybe?).
// To avoid 0/0, set the fraction to 0.
hitFraction = castHit ? (scaledRayLength > 0 ? rayHit.distance / scaledRayLength : 0.0f) : 1.0f;
hitObject = castHit ? rayHit.collider.gameObject : null;
#endif
}
else
{
#if MLA_UNITY_PHYSICS2D_MODULE
RaycastHit2D rayHit;
if (scaledCastRadius > 0f)
{
rayHit = Physics2D.CircleCast(startPositionWorld, scaledCastRadius, rayDirection,
scaledRayLength, input.LayerMask);
}
else
{
rayHit = Physics2D.Raycast(startPositionWorld, rayDirection, scaledRayLength, input.LayerMask);
}
castHit = rayHit;
hitFraction = castHit ? rayHit.fraction : 1.0f;
hitObject = castHit ? rayHit.collider.gameObject : null;
#endif
}
var rayOutput = new RayPerceptionOutput.RayOutput
{
HasHit = castHit,
HitFraction = hitFraction,
HitTaggedObject = false,
HitTagIndex = -1,
HitGameObject = hitObject,
StartPositionWorld = startPositionWorld,
EndPositionWorld = endPositionWorld,
ScaledCastRadius = scaledCastRadius
};
if (castHit)
{
// Find the index of the tag of the object that was hit.
var numTags = input.DetectableTags?.Count ?? 0;
for (var i = 0; i < numTags; i++)
{
var tagsEqual = false;
try
{
var tag = input.DetectableTags[i];
if (!string.IsNullOrEmpty(tag))
{
tagsEqual = hitObject.CompareTag(tag);
}
}
catch (UnityException)
{
// If the tag is null, empty, or not a valid tag, just ignore it.
}
if (tagsEqual)
{
rayOutput.HitTaggedObject = true;
rayOutput.HitTagIndex = i;
break;
}
}
}
return rayOutput;
}
}
}