浏览代码

WIP dump - working prototype

Updates to CameraSensor / CameraSensorComponent:
* new settings object to set up layer masks, depth, etc
* support for arbitrary # channels returned by the sensor
* channels include depth, layer mask, RGB, or grayscale

Updates to VisualFoodCollector:
* food doesn't roll (for now), simplifies learning without stacking
* added layers

Updates to mlagents_envs:
* decode multiple pngs packed into a single observation
/hw20-segmentation
Jonathan Harper 4 年前
当前提交
5906cd61
共有 24 个文件被更改,包括 2058 次插入905 次删除
  1. 92
      Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/BadFood.prefab
  2. 92
      Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/Food.prefab
  3. 854
      Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/VisualFoodCollectorArea.prefab
  4. 9
      Project/Assets/ML-Agents/Examples/FoodCollector/Scenes/VisualFoodCollector.unity
  5. 24
      Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/Area.prefab
  6. 318
      Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/goal.prefab
  7. 326
      Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/pit.prefab
  8. 5
      Project/ProjectSettings/EditorBuildSettings.asset
  9. 2
      Project/ProjectSettings/ProjectVersion.txt
  10. 16
      Project/ProjectSettings/TagManager.asset
  11. 1
      com.unity.ml-agents/Editor/CameraSensorComponentEditor.cs
  12. 481
      com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs
  13. 24
      com.unity.ml-agents/Runtime/Sensors/CameraSensorComponent.cs
  14. 28
      ml-agents-envs/mlagents_envs/rpc_utils.py
  15. 105
      com.unity.ml-agents/Editor/CameraSensorSettingsDrawer.cs
  16. 3
      com.unity.ml-agents/Editor/CameraSensorSettingsDrawer.cs.meta
  17. 37
      com.unity.ml-agents/Runtime/Sensors/CameraSensorSettings.cs
  18. 3
      com.unity.ml-agents/Runtime/Sensors/CameraSensorSettings.cs.meta
  19. 83
      com.unity.ml-agents/Runtime/Sensors/ColorEncoding.cs
  20. 3
      com.unity.ml-agents/Runtime/Sensors/ColorEncoding.cs.meta
  21. 8
      com.unity.ml-agents/Runtime/Shaders.meta
  22. 440
      com.unity.ml-agents/Runtime/Shaders/UberReplacement.shader
  23. 9
      com.unity.ml-agents/Runtime/Shaders/UberReplacement.shader.meta

92
Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/BadFood.prefab


%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1001 &100100000
Prefab:
m_ObjectHideFlags: 1
serializedVersion: 2
m_Modification:
m_TransformParent: {fileID: 0}
m_Modifications: []
m_RemovedComponents: []
m_ParentPrefab: {fileID: 0}
m_RootGameObject: {fileID: 1853821981215314}
m_IsPrefabParent: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4094284922894548}
- component: {fileID: 33861055306973786}

- component: {fileID: 114054540722299252}
m_Layer: 0
m_Layer: 12
m_Name: BadFood
m_TagString: badFood
m_Icon: {fileID: 0}

--- !u!4 &4094284922894548
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -32.01, y: 2, z: -12.46}

m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &33861055306973786
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
m_Mesh: {fileID: 10207, guid: 0000000000000000e000000000000000, type: 0}
--- !u!135 &135398688903115902
SphereCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Radius: 0.75
m_Center: {x: 0, y: 0, z: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
m_Enabled: 1
m_CastShadows: 1

m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 88b9ae7af2c1748a0a1f63407587a601, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &33861055306973786
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1853821981215314}
m_Mesh: {fileID: 10207, guid: 0000000000000000e000000000000000, type: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
serializedVersion: 2
m_Mass: 5

m_IsKinematic: 0
m_Interpolate: 0
m_Constraints: 0
m_Constraints: 90
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1853821981215314}
m_Enabled: 1
m_EditorHideFlags: 0

respawn: 0
myArea: {fileID: 0}
--- !u!135 &135398688903115902
SphereCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1853821981215314}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Radius: 0.75
m_Center: {x: 0, y: 0, z: 0}

92
Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/Food.prefab


%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1001 &100100000
Prefab:
m_ObjectHideFlags: 1
serializedVersion: 2
m_Modification:
m_TransformParent: {fileID: 0}
m_Modifications: []
m_RemovedComponents: []
m_ParentPrefab: {fileID: 0}
m_RootGameObject: {fileID: 1699568242032636}
m_IsPrefabParent: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4317744128851916}
- component: {fileID: 33865088620820440}

- component: {fileID: 114154930672833238}
m_Layer: 0
m_Layer: 13
m_Name: Food
m_TagString: food
m_Icon: {fileID: 0}

--- !u!4 &4317744128851916
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: -32.01, y: 2, z: -9.559753}

m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &33865088620820440
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
m_Mesh: {fileID: 10207, guid: 0000000000000000e000000000000000, type: 0}
--- !u!135 &135161629589834772
SphereCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Radius: 0.75
m_Center: {x: 0, y: 0, z: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
m_Enabled: 1
m_CastShadows: 1

m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: c67450f290f3e4897bc40276a619e78d, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &33865088620820440
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1699568242032636}
m_Mesh: {fileID: 10207, guid: 0000000000000000e000000000000000, type: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
serializedVersion: 2
m_Mass: 5

m_IsKinematic: 0
m_Interpolate: 0
m_Constraints: 0
m_Constraints: 90
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1699568242032636}
m_Enabled: 1
m_EditorHideFlags: 0

respawn: 0
myArea: {fileID: 0}
--- !u!135 &135161629589834772
SphereCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1699568242032636}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 2
m_Radius: 0.75
m_Center: {x: 0, y: 0, z: 0}

854
Project/Assets/ML-Agents/Examples/FoodCollector/Prefabs/VisualFoodCollectorArea.prefab
文件差异内容过多而无法显示
查看文件

9
Project/Assets/ML-Agents/Examples/FoodCollector/Scenes/VisualFoodCollector.unity


m_Modification:
m_TransformParent: {fileID: 0}
m_Modifications:
- target: {fileID: 1537641056927260, guid: 3ce107b4a79bc4eef83afde434932a68, type: 3}
propertyPath: m_IsActive
value: 0
objectReference: {fileID: 0}
- target: {fileID: 224194346362733190, guid: 3ce107b4a79bc4eef83afde434932a68,
type: 3}
propertyPath: m_LocalPosition.x

- target: {fileID: 4307641258646068, guid: c85b585836e104587b4efdc4d8b9d62b, type: 3}
propertyPath: m_RootOrder
value: 5
objectReference: {fileID: 0}
- target: {fileID: 20696931947702132, guid: c85b585836e104587b4efdc4d8b9d62b,
type: 3}
propertyPath: m_TargetDisplay
value: 1
objectReference: {fileID: 0}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: c85b585836e104587b4efdc4d8b9d62b, type: 3}

24
Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/Area.prefab


- component: {fileID: 33152597024021804}
- component: {fileID: 65829039252076494}
- component: {fileID: 23027717151182946}
m_Layer: 0
m_Layer: 11
m_Name: Cube (1)
m_TagString: Untagged
m_Icon: {fileID: 0}

- component: {fileID: 114650561397225712}
- component: {fileID: 114889700908650620}
- component: {fileID: 7980686505185502968}
m_Layer: 8
m_Layer: 11
m_Name: Agent
m_TagString: agent
m_Icon: {fileID: 0}

m_Name:
m_EditorClassIdentifier:
m_BrainParameters:
vectorObservationSize: 0
numStackedVectorObservations: 1
vectorActionSize: 05000000
vectorActionDescriptions: []
vectorActionSpaceType: 0
VectorObservationSize: 0
NumStackedVectorObservations: 1
VectorActionSize: 05000000
VectorActionDescriptions: []
VectorActionSpaceType: 0
m_Model: {fileID: 11400000, guid: a812f1ce7763a4a0c912717f3594fe20, type: 3}
m_InferenceDevice: 0
m_BehaviorType: 0

m_ObservableAttributeHandling: 0
--- !u!114 &114650561397225712
MonoBehaviour:
m_ObjectHideFlags: 0

agentParameters:
maxStep: 0
hasUpgradedFromAgentParameters: 1
maxStep: 100
MaxStep: 100
area: {fileID: 114704252266302846}
timeBetweenDecisionsAtInference: 0.15
renderCamera: {fileID: 0}

m_Script: {fileID: 11500000, guid: 3a6da8f78a394c6ab027688eab81e04d, type: 3}
m_Name:
m_EditorClassIdentifier:
debugCommandLineOverride:
--- !u!1 &1625008366184734
GameObject:
m_ObjectHideFlags: 0

- component: {fileID: 33813191588135886}
- component: {fileID: 65922560747716658}
- component: {fileID: 23740566138369150}
m_Layer: 0
m_Layer: 11
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}

- component: {fileID: 33825707296015202}
- component: {fileID: 65020466233620882}
- component: {fileID: 23205928308810388}
m_Layer: 0
m_Layer: 11
m_Name: Cube (3)
m_TagString: Untagged
m_Icon: {fileID: 0}

- component: {fileID: 33172111281575160}
- component: {fileID: 65321351298141546}
- component: {fileID: 23200653194041678}
m_Layer: 0
m_Layer: 11
m_Name: Cube (2)
m_TagString: Untagged
m_Icon: {fileID: 0}

318
Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/goal.prefab


%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1001 &100100000
Prefab:
m_ObjectHideFlags: 1
serializedVersion: 2
m_Modification:
m_TransformParent: {fileID: 0}
m_Modifications: []
m_RemovedComponents: []
m_ParentPrefab: {fileID: 0}
m_RootGameObject: {fileID: 1508142483324970}
m_IsPrefabParent: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Layer: 0
m_Layer: 10
m_Name: goal
m_TagString: goal
m_Icon: {fileID: 0}

--- !u!1 &1765554778432800
GameObject:
m_ObjectHideFlags: 0
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_Component:
- component: {fileID: 4401303026649678}
- component: {fileID: 33048545948983598}
- component: {fileID: 65991532807116388}
- component: {fileID: 23889676296172340}
m_Layer: 0
m_Name: Cube (1)
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!1 &1823577511622086
GameObject:
m_ObjectHideFlags: 0
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_Component:
- component: {fileID: 4889404563930382}
- component: {fileID: 33298529236581638}
- component: {fileID: 65392370003653392}
- component: {fileID: 23079780473834094}
m_Layer: 0
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1508142483324970}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}

m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!4 &4401303026649678
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1765554778432800}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0.2, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 4266498611226432}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!4 &4889404563930382
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1823577511622086}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 0.2}
m_Children: []
m_Father: {fileID: 4266498611226432}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!23 &23079780473834094
--- !u!33 &33776711028771674
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1508142483324970}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65981801441420474
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1508142483324970}
m_Material: {fileID: 0}
m_IsTrigger: 1
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 0.9, y: 0.9, z: 0.9}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &23936659159756770
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1823577511622086}
m_Enabled: 1
m_CastShadows: 1
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1508142483324970}
m_Enabled: 0
m_CastShadows: 0
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: c67450f290f3e4897bc40276a619e78d, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!1 &1765554778432800
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4401303026649678}
- component: {fileID: 33048545948983598}
- component: {fileID: 65991532807116388}
- component: {fileID: 23889676296172340}
m_Layer: 10
m_Name: Cube (1)
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &4401303026649678
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1765554778432800}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0.2, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 4266498611226432}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &33048545948983598
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1765554778432800}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65991532807116388
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1765554778432800}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1765554778432800}
m_Enabled: 1
m_CastShadows: 1

m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: c67450f290f3e4897bc40276a619e78d, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!23 &23936659159756770
MeshRenderer:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1508142483324970}
--- !u!1 &1823577511622086
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4889404563930382}
- component: {fileID: 33298529236581638}
- component: {fileID: 65392370003653392}
- component: {fileID: 23079780473834094}
m_Layer: 10
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &4889404563930382
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1823577511622086}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 0.2}
m_Children: []
m_Father: {fileID: 4266498611226432}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &33298529236581638
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1823577511622086}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65392370003653392
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1823577511622086}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_CastShadows: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &23079780473834094
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1823577511622086}
m_Enabled: 1
m_CastShadows: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: c67450f290f3e4897bc40276a619e78d, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &33048545948983598
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1765554778432800}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!33 &33298529236581638
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1823577511622086}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!33 &33776711028771674
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1508142483324970}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65392370003653392
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1823577511622086}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!65 &65981801441420474
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1508142483324970}
m_Material: {fileID: 0}
m_IsTrigger: 1
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 0.9, y: 0.9, z: 0.9}
m_Center: {x: 0, y: 0, z: 0}
--- !u!65 &65991532807116388
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1765554778432800}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}

326
Project/Assets/ML-Agents/Examples/GridWorld/Prefabs/pit.prefab


%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1001 &100100000
Prefab:
m_ObjectHideFlags: 1
serializedVersion: 2
m_Modification:
m_TransformParent: {fileID: 0}
m_Modifications: []
m_RemovedComponents: []
m_ParentPrefab: {fileID: 0}
m_RootGameObject: {fileID: 1811317785436014}
m_IsPrefabParent: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Layer: 0
m_Layer: 9
m_Name: Cube (1)
m_TagString: Untagged
m_Icon: {fileID: 0}

--- !u!1 &1778744900735866
GameObject:
m_ObjectHideFlags: 0
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_Component:
- component: {fileID: 4012045820321770}
- component: {fileID: 33355843709390916}
- component: {fileID: 65087511814012390}
- component: {fileID: 23151230937589412}
m_Layer: 0
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!1 &1811317785436014
GameObject:
m_ObjectHideFlags: 0
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
serializedVersion: 5
m_Component:
- component: {fileID: 4302184315941402}
- component: {fileID: 33217562963522382}
- component: {fileID: 65172047905851350}
- component: {fileID: 23740670609665506}
m_Layer: 0
m_Name: pit
m_TagString: pit
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &4012045820321770
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1778744900735866}
m_LocalRotation: {x: -0, y: 0.38268343, z: -0, w: 0.92387956}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0.20000002, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 4302184315941402}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 45, z: 0}
--- !u!4 &4302184315941402
Transform:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1811317785436014}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children:
- {fileID: 4012045820321770}
- {fileID: 4873889445063974}
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1308998153656692}
m_LocalRotation: {x: -0, y: -0.38268343, z: -0, w: 0.92387956}
m_LocalPosition: {x: 0, y: 0, z: 0}

m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: -45, z: 0}
--- !u!23 &23151230937589412
--- !u!33 &33297748647671602
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1308998153656692}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65439704505015706
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1308998153656692}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &23833739472231758
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1778744900735866}
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1308998153656692}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1

m_ReflectionProbeUsage: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 88b9ae7af2c1748a0a1f63407587a601, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!23 &23740670609665506
MeshRenderer:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1811317785436014}
--- !u!1 &1778744900735866
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4012045820321770}
- component: {fileID: 33355843709390916}
- component: {fileID: 65087511814012390}
- component: {fileID: 23151230937589412}
m_Layer: 9
m_Name: Cube
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &4012045820321770
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1778744900735866}
m_LocalRotation: {x: -0, y: 0.38268343, z: -0, w: 0.92387956}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 0.20000002, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 4302184315941402}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 45, z: 0}
--- !u!33 &33355843709390916
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1778744900735866}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65087511814012390
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1778744900735866}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_CastShadows: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &23151230937589412
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1778744900735866}
m_Enabled: 1
m_CastShadows: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 88b9ae7af2c1748a0a1f63407587a601, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!23 &23833739472231758
--- !u!1 &1811317785436014
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4302184315941402}
- component: {fileID: 33217562963522382}
- component: {fileID: 65172047905851350}
- component: {fileID: 23740670609665506}
m_Layer: 9
m_Name: pit
m_TagString: pit
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &4302184315941402
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1811317785436014}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children:
- {fileID: 4012045820321770}
- {fileID: 4873889445063974}
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!33 &33217562963522382
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1811317785436014}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65172047905851350
BoxCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1811317785436014}
m_Material: {fileID: 0}
m_IsTrigger: 1
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 0.9, y: 0.9, z: 0.9}
m_Center: {x: 0, y: 0, z: 0}
--- !u!23 &23740670609665506
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1308998153656692}
m_Enabled: 1
m_CastShadows: 1
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1811317785436014}
m_Enabled: 0
m_CastShadows: 0
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 2100000, guid: 88b9ae7af2c1748a0a1f63407587a601, type: 2}
m_StaticBatchInfo:

m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &33217562963522382
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1811317785436014}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!33 &33297748647671602
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1308998153656692}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!33 &33355843709390916
MeshFilter:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1778744900735866}
m_Mesh: {fileID: 10202, guid: 0000000000000000e000000000000000, type: 0}
--- !u!65 &65087511814012390
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1778744900735866}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}
--- !u!65 &65172047905851350
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1811317785436014}
m_Material: {fileID: 0}
m_IsTrigger: 1
m_Enabled: 1
serializedVersion: 2
m_Size: {x: 0.9, y: 0.9, z: 0.9}
m_Center: {x: 0, y: 0, z: 0}
--- !u!65 &65439704505015706
BoxCollider:
m_ObjectHideFlags: 1
m_PrefabParentObject: {fileID: 0}
m_PrefabInternal: {fileID: 100100000}
m_GameObject: {fileID: 1308998153656692}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 0
serializedVersion: 2
m_Size: {x: 1, y: 1, z: 1}
m_Center: {x: 0, y: 0, z: 0}

5
Project/ProjectSettings/EditorBuildSettings.asset


EditorBuildSettings:
m_ObjectHideFlags: 0
serializedVersion: 2
m_Scenes: []
m_Scenes:
- enabled: 1
path: Assets/ML-Agents/Examples/FoodCollector/Scenes/VisualFoodCollector.unity
guid: dbc5c542957ef47bd8ebe87fc1000c37
m_configObjects: {}

2
Project/ProjectSettings/ProjectVersion.txt


m_EditorVersion: 2018.4.17f1
m_EditorVersion: 2018.4.18f1

16
Project/ProjectSettings/TagManager.asset


-
-
- invisible
-
-
-
-
-
-
-
-
- pit
- goal
- agent
- badFood
- food
- floor
- walls
- laser
-
-
-

1
com.unity.ml-agents/Editor/CameraSensorComponentEditor.cs


EditorGUILayout.PropertyField(so.FindProperty("m_Width"), true);
EditorGUILayout.PropertyField(so.FindProperty("m_Height"), true);
EditorGUILayout.PropertyField(so.FindProperty("m_Grayscale"), true);
EditorGUILayout.PropertyField(so.FindProperty("m_CameraSensorSettings"), true);
}
EditorGUI.EndDisabledGroup();
EditorGUILayout.PropertyField(so.FindProperty("m_Compression"), true);

481
com.unity.ml-agents/Runtime/Sensors/CameraSensor.cs


using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using UnityEngine.Rendering;
using Object = UnityEngine.Object;
internal enum CameraSensorPassType
{
RGB,
Depth,
Segmentation
}
struct CameraSensorPass
{
public CameraSensorPassType PassType;
public Camera Camera;
public CameraSensorPass(CameraSensorPassType passType, Camera camera)
{
PassType = passType;
Camera = camera;
}
}
internal enum CameraChannelType
{
RGB,
Grayscale,
Depth,
OpticalFlow,
LayerMask
}
internal struct CameraSensorChannel
{
public CameraChannelType ChannelType;
public int? LayerNumber;
public CameraSensorChannel(CameraChannelType channelType, int? layerNumber = null)
{
ChannelType = channelType;
LayerNumber = layerNumber;
}
}
/// <summary>
/// A sensor that wraps a Camera object to generate visual observations for an agent.
/// </summary>

string m_Name;
int[] m_Shape;
SensorCompressionType m_CompressionType;
Shader uberReplacementShader = Shader.Find("Hidden/UberReplacement");
static readonly int _objectColor = Shader.PropertyToID("_ObjectColor");
static readonly int _categoryColor = Shader.PropertyToID("_CategoryColor");
static readonly int _layerNumber = Shader.PropertyToID("_LayerNumber");
CameraSensorSettings m_Settings;
CommandBuffer m_AddedBuffer;
List<CameraSensorPass> m_Passes;
/// <summary>
/// The Camera used for rendering the sensor observations.

set { m_Camera = value; }
}
internal List<CameraSensorPass> CreatePasses()
{
var passes = new List<CameraSensorPass>();
if (!m_Settings.DisableCamera)
{
passes.Add(new CameraSensorPass(CameraSensorPassType.RGB, m_Camera));
}
if (m_Settings.EnableDepth)
{
passes.Add(new CameraSensorPass(CameraSensorPassType.Depth, CreateHiddenCamera("depthCam")));
}
if (m_Settings.LayerMasks.Length > 0)
{
passes.Add(new CameraSensorPass(CameraSensorPassType.Segmentation, CreateHiddenCamera("segmentationCam")));
}
return passes;
}
Camera CreateHiddenCamera(string name)
{
var go = new GameObject (name, typeof (Camera));
go.hideFlags = HideFlags.HideAndDontSave;
go.transform.parent = m_Camera.transform;
var newCamera = go.GetComponent<Camera>();
return newCamera;
}
/// <summary>
/// The compression type used by the sensor.
/// </summary>

set { m_CompressionType = value; }
}
internal List<CameraSensorChannel> Channels()
{
var channels = new List<CameraSensorChannel>();
if (!m_Settings.DisableCamera)
{
if (m_Grayscale)
{
channels.Add(new CameraSensorChannel(CameraChannelType.Grayscale));
}
else
{
channels.Add(new CameraSensorChannel(CameraChannelType.RGB));
}
}
if (m_Settings.EnableDepth)
{
channels.Add(new CameraSensorChannel(CameraChannelType.Depth));
}
if (m_Settings.LayerMasks.Length > 0)
{
foreach (var layerToMask in m_Settings.LayerMasks)
{
channels.Add(
new CameraSensorChannel(CameraChannelType.LayerMask, layerToMask)
);
}
}
return channels;
}
/// <summary>
/// Creates and returns the camera sensor.

public CameraSensor(
Camera camera, int width, int height, bool grayscale, string name, SensorCompressionType compression)
{
m_Camera = camera;
m_Settings = new CameraSensorSettings();
Camera = camera;
m_Width = width;
m_Height = height;
m_Grayscale = grayscale;
m_Name = name;
m_CompressionType = compression;
m_Passes = CreatePasses();
m_Shape = GenerateShape();
OnSceneChange();
}
public CameraSensor(
Camera camera,
int width,
int height,
bool grayscale,
string name,
SensorCompressionType compression,
CameraSensorSettings settings
) : this(camera, width, height, grayscale, name, compression)
{
m_Settings = settings;
Camera = camera;
m_Shape = GenerateShape(width, height, grayscale);
m_Passes = CreatePasses();
m_Shape = GenerateShape();
OnSceneChange();
}
public void OnSceneChange()
{
var segPasses = m_Passes.Where(
p => p.PassType == CameraSensorPassType.Segmentation
);
// NOTE: This avoids the expensive search for all renderers if we don't need segmentation.
if (!segPasses.Any()) return;
var renderers = Object.FindObjectsOfType<Renderer>();
// var renderers = m_Camera.transform.root.GetComponentsInChildren<Renderer>();
var mpb = new MaterialPropertyBlock();
foreach (var r in renderers)
{
GameObject gameObject = r.gameObject;
var id = gameObject.GetInstanceID();
var layer = gameObject.layer;
var tag = gameObject.tag;
mpb.SetColor(_objectColor, ColorEncoding.EncodeIDAsColor(id));
mpb.SetColor(_categoryColor, ColorEncoding.EncodeLayerAsColor(layer));
mpb.SetInt(_layerNumber, layer);
r.SetPropertyBlock(mpb);
}
}
internal List<float[,]> GetObservationChannels()
{
UpdateCameras();
OnSceneChange();
var channels = new List<float[,]>();
foreach (var pass in m_Passes)
{
var disableAntialiasing = pass.PassType != CameraSensorPassType.RGB;
var texture = ObservationToTexture(pass.Camera, m_Width, m_Height, disableAntialiasing);
var width = texture.width;
var height = texture.height;
var texturePixels = texture.GetPixels();
if (pass.PassType == CameraSensorPassType.RGB)
{
var r = new float[texture.width, texture.height];
var g = new float[texture.width, texture.height];
var b = new float[texture.width, texture.height];
for (var h = height - 1; h >= 0; h--)
{
for (var w = 0; w < width; w++)
{
var currentPixel = texturePixels[(height - h - 1) * width + w];
if (m_Grayscale)
{
r[h, w] = (currentPixel.r + currentPixel.g + currentPixel.b) / 3f;
}
else
{
// For Color32, the r, g and b values are between 0 and 1.
r[h, w] = currentPixel.r;
g[h, w] = currentPixel.g;
b[h, w] = currentPixel.b;
}
}
}
if (m_Grayscale)
{
channels.Add(r);
}
else
{
channels.Add(r); channels.Add(g); channels.Add(b);
}
}
else if (pass.PassType == CameraSensorPassType.Depth)
{
var depth = new float[texture.width, texture.height];
for (var h = height - 1; h >= 0; h--)
{
for (var w = 0; w < width; w++)
{
var currentPixel = texturePixels[(height - h - 1) * width + w];
depth[h, w] = (currentPixel.r + currentPixel.g + currentPixel.b) / 3f;
}
}
channels.Add(depth);
}
else if (pass.PassType == CameraSensorPassType.Segmentation)
{
var loggedInThisPass = false;
var layerChannels = new List<float[,]>();
foreach (var layer in m_Settings.LayerMasks)
{
layerChannels.Add(new float[texture.width, texture.height]);
}
for (var h = height - 1; h >= 0; h--)
{
for (var w = 0; w < width; w++)
{
var currentPixel = texturePixels[(height - h - 1) * width + w];
// layer index is encoded in the red channel, divided by 100
var currentPixelIndex = (int)Math.Round((double) currentPixel.r * 100f);
// Try to decode the "r" channel as one of our layer IDs
var foundLayerIndex = Array.FindIndex(m_Settings.LayerMasks, i => i == currentPixelIndex);
if (foundLayerIndex >= 0)
{
// We found it, so set the value to 1
var layerChannel = layerChannels[foundLayerIndex];
layerChannel[h, w] = 1f;
}
}
}
foreach (var layerChannel in layerChannels) channels.Add(layerChannel);
}
DestroyTexture(texture);
}
return channels;
}
// Texture2D MakeGrayscale (tex : Texture2D) {
// var texColors = tex.GetPixels();
// for (i = 0; i < texColors.Length; i++) {
// var grayValue = texColors[i].grayscale;
// texColors[i] = Color(grayValue, grayValue, grayValue, texColors[i].a);
// }
// tex.SetPixels(texColors);
// tex.Apply();
// }
internal List<Texture2D> GetObservationTextures()
{
UpdateCameras();
OnSceneChange();
var textures = new List<Texture2D>();
var singleChannels = new List<float[]>();
foreach (var pass in m_Passes)
{
var disableAntialiasing = pass.PassType != CameraSensorPassType.RGB;
var texture = ObservationToTexture(pass.Camera, m_Width, m_Height, disableAntialiasing);
var width = texture.width;
var height = texture.height;
var texturePixels = texture.GetPixels();
if (pass.PassType == CameraSensorPassType.RGB)
{
if (m_Grayscale)
{
// turn to grayscale
for (var i = 0; i < texturePixels.Length; ++i)
{
var grayVal = texturePixels[i].grayscale;
texturePixels[i] = new Color(grayVal, grayVal, grayVal, texturePixels[i].a);
}
texture.SetPixels(texturePixels);
texture.Apply();
textures.Add(texture);
}
else
{
textures.Add(texture);
}
}
else if (pass.PassType == CameraSensorPassType.Depth)
{
// single channel -- convert to grayscale and add to single channels list
var channelVals = new float[texturePixels.Length];
for (var i = 0; i < texturePixels.Length; ++i)
{
channelVals[i] = texturePixels[i].grayscale;
}
singleChannels.Add(channelVals);
}
else if (pass.PassType == CameraSensorPassType.Segmentation)
{
var layerChannels = new List<float[]>();
foreach (var layer in m_Settings.LayerMasks)
{
layerChannels.Add(new float[texturePixels.Length]);
}
for (var i = 0; i < texturePixels.Length; ++i)
{
var currentPixel = texturePixels[i];
// layer index is encoded in the red channel, divided by 100
var currentPixelIndex = (int)Math.Round((double) currentPixel.r * 100f);
// Try to decode the "r" channel as one of our layer IDs
var foundLayerIndex = Array.FindIndex(m_Settings.LayerMasks, j => j == currentPixelIndex);
if (foundLayerIndex >= 0)
{
// We found it, so set the value to 1
var layerChannel = layerChannels[foundLayerIndex];
layerChannel[i] = 1f;
}
}
foreach (var layerChannel in layerChannels) singleChannels.Add(layerChannel);
}
}
// Single channels need to be converted to colors and assigned to new textures.
// if we have 5 channels, we will need ceil(5/3) textures. Or, 2.
var numNewTextures = (int) Math.Ceiling(singleChannels.Count / 3f);
for (var i = 0; i < numNewTextures; ++i)
{
var tex = new Texture2D(m_Width, m_Height, TextureFormat.RGB24, false);
var colors = new Color[m_Width * m_Height];
var begin = i * 3;
for (var j = 0; j < m_Width * m_Height; ++j)
{
var firstChannelVal = singleChannels[begin][j];
var secondChannelVal = 0f;
if (singleChannels.Count > begin + 1)
{
secondChannelVal = singleChannels[begin + 1][j];
}
var thirdChannelVal = 0f;
if (singleChannels.Count > begin + 2)
{
thirdChannelVal = singleChannels[begin + 2][j];
}
colors[j] = new Color(firstChannelVal, secondChannelVal, thirdChannelVal);
}
tex.SetPixels(colors);
tex.Apply();
textures.Add(tex);
}
return textures;
}
public void UpdateCameras()
{
foreach (var pass in m_Passes)
{
if (pass.PassType == CameraSensorPassType.RGB)
continue;
// cleanup capturing camera
pass.Camera.RemoveAllCommandBuffers();
// copy all "main" camera parameters into capturing camera
pass.Camera.CopyFrom(m_Camera);
// setup command buffers and replacement shaders
SetupCameraWithReplacementShader(pass.Camera, uberReplacementShader, pass.PassType);
}
// cache materials and setup material properties
// if (!opticalFlowMaterial || opticalFlowMaterial.shader != opticalFlowShader)
// opticalFlowMaterial = new Material(opticalFlowShader);
// opticalFlowMaterial.SetFloat("_Sensitivity", opticalFlowSensitivity);
}
static void SetupCameraWithReplacementShader(Camera cam, Shader shader, CameraSensorPassType passType)
{
var cb = new CommandBuffer();
Color clearColor = Color.black;
int mode = -1;
if (passType == CameraSensorPassType.Depth)
{
mode = 2;
clearColor = Color.white;
}
else if (passType == CameraSensorPassType.Segmentation)
{
//Debug.Log("Setting shader mode to 3");
mode = 3;
}
if (mode < 0)
{
throw new Exception("Error: unexpected pass type");
}
cb.SetGlobalInt("_OutputMode", mode);
cam.AddCommandBuffer(CameraEvent.BeforeForwardOpaque, cb);
cam.AddCommandBuffer(CameraEvent.BeforeFinalPass, cb);
cam.SetReplacementShader(shader, "");
cam.backgroundColor = clearColor;
cam.clearFlags = CameraClearFlags.SolidColor;
}
/// <summary>

/// <returns>Compressed image.</returns>
public byte[] GetCompressedObservation()
{
var allBytes = new List<byte>();
var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
var textures = GetObservationTextures();
var compressed = texture.EncodeToPNG();
DestroyTexture(texture);
return compressed;
foreach (var tex in textures)
{
allBytes.AddRange(tex.EncodeToPNG());
DestroyTexture(tex);
}
return allBytes.ToArray();
}
}

{
using (TimerStack.Instance.Scoped("CameraSensor.WriteToTensor"))
{
var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
var numWritten = Utilities.TextureToTensorProxy(texture, writer, m_Grayscale);
DestroyTexture(texture);
//var texture = ObservationToTexture(m_Camera, m_Width, m_Height);
//var numWritten = Utilities.TextureToTensorProxy(texture, writer, m_Grayscale);
//DestroyTexture(texture);
var numWritten = 0;
var channels = GetObservationChannels();
for (var channelInd = 0; channelInd < channels.Count; ++channelInd)
{
for (var h = 0; h < m_Height; ++h)
{
for (var w = 0; w < m_Width; ++w)
{
writer[h, w, channelInd] = channels[channelInd][h, w];
numWritten++;
}
}
}
return numWritten;
}
}

/// <param name="width">Width of resulting 2D texture.</param>
/// <param name="height">Height of resulting 2D texture.</param>
/// <returns name="texture2D">Texture2D to render to.</returns>
public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height)
public static Texture2D ObservationToTexture(Camera obsCamera, int width, int height, bool disableAntialiasing = false)
{
var texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
var oldRec = obsCamera.rect;

var readWrite = RenderTextureReadWrite.Default;
var antiAliasing = (disableAntialiasing) ? 1 : Mathf.Max(1, QualitySettings.antiAliasing);
RenderTexture.GetTemporary(width, height, depth, format, readWrite);
RenderTexture.GetTemporary(width, height, depth, format, readWrite, antiAliasing);
var prevActiveRt = RenderTexture.active;
var prevCameraRt = obsCamera.targetTexture;

obsCamera.Render();
texture2D.ReadPixels(new Rect(0, 0, texture2D.width, texture2D.height), 0, 0);
texture2D.Apply();
obsCamera.targetTexture = prevCameraRt;
obsCamera.rect = oldRec;

/// <param name="height">Height of the image captures from the camera.</param>
/// <param name="grayscale">Whether or not to convert the image to grayscale.</param>
/// <returns>The observation shape.</returns>
internal static int[] GenerateShape(int width, int height, bool grayscale)
internal int[] GenerateShape()
return new[] { height, width, grayscale ? 1 : 3 };
int channels = 0;
foreach (var pass in m_Passes)
{
if (pass.PassType == CameraSensorPassType.RGB)
{
if (m_Grayscale) channels += 1;
else channels += 3;
}
if (pass.PassType == CameraSensorPassType.Depth)
{
channels += 1;
}
if (pass.PassType == CameraSensorPassType.Segmentation)
{
channels += m_Settings.LayerMasks.Length;
}
}
return new[] { m_Height, m_Width, channels };
}
static void DestroyTexture(Texture2D texture)

24
com.unity.ml-agents/Runtime/Sensors/CameraSensorComponent.cs


set { m_Grayscale = value; }
}
[HideInInspector, SerializeField]
CameraSensorSettings m_CameraSensorSettings = new CameraSensorSettings();
/// <summary>
/// Whether to render the main camera.
/// </summary>
public CameraSensorSettings CameraSensorSettings
{
get { return m_CameraSensorSettings; }
set { m_CameraSensorSettings = value; }
}
/// <summary>
///
/// </summary>
[HideInInspector, SerializeField, FormerlySerializedAs("compression")]
SensorCompressionType m_Compression = SensorCompressionType.PNG;

/// <returns>The created <see cref="CameraSensor"/> object for this component.</returns>
public override ISensor CreateSensor()
{
m_Sensor = new CameraSensor(m_Camera, m_Width, m_Height, Grayscale, m_SensorName, m_Compression);
m_Sensor = new CameraSensor(m_Camera, m_Width, m_Height, Grayscale, m_SensorName, m_Compression, m_CameraSensorSettings);
return m_Sensor;
}

/// <returns>The observation shape of the associated <see cref="CameraSensor"/> object.</returns>
public override int[] GetObservationShape()
{
return CameraSensor.GenerateShape(m_Width, m_Height, Grayscale);
if (m_Sensor == null)
{
CreateSensor();
}
return m_Sensor.GenerateShape();
}
/// <summary>

28
ml-agents-envs/mlagents_envs/rpc_utils.py


:param image_bytes: input byte array corresponding to image
:return: processed numpy array of observation from environment
"""
# we may have multiple pngs, so first
png_header = b"\x89PNG\r\n\x1a\n"
images = []
image = Image.open(io.BytesIO(image_bytearray))
# Normally Image loads lazily, this forces it to do loading in the timer scope.
image.load()
s = np.array(image, dtype=np.float32) / 255.0
if gray_scale:
s = np.mean(s, axis=2)
s = np.reshape(s, [s.shape[0], s.shape[1], 1])
return s
image_byte_splits = image_bytearray.split(png_header)
for image_bytes in image_byte_splits:
if len(image_bytes) > 0:
# add back the header
image = Image.open(io.BytesIO(png_header + image_bytes))
# Normally Image loads lazily, this forces it to do loading in the timer scope.
image.load()
images.append(image)
arrs = list(map(lambda i: np.array(i, dtype=np.float32) / 255.0, images))
return np.concatenate(arrs, axis=2)
@timed

return img
else:
img = process_pixels(obs.compressed_data, gray_scale)
# We can drop additional channels since they may need to be added to include
# numbers of observation channels not divisible by 3.
expected_channels = list(obs.shape)[2]
actual_channels = list(img.shape)[2]
if actual_channels > expected_channels:
img = img[..., 0:expected_channels]
# Compare decompressed image size to observation shape and make sure they match
if list(obs.shape) != list(img.shape):
raise UnityObservationException(

105
com.unity.ml-agents/Editor/CameraSensorSettingsDrawer.cs


using Unity.MLAgents.Sensors;
using UnityEditor;
using UnityEngine;
namespace Unity.MLAgents.Editor
{
/// <summary>
/// PropertyDrawer for BrainParameters. Defines how BrainParameters are displayed in the
/// Inspector.
/// </summary>
[CustomPropertyDrawer(typeof(CameraSensorSettings))]
internal class CameraSensorSettingsDrawer : PropertyDrawer
{
// The height of a line in the Unity Inspectors
const float k_LineHeight = 17f;
const string k_LayerMasksPropName = "LayerMasks";
const string k_DisableCameraPropName = "DisableCamera";
const string k_EnableDepthPropName = "EnableDepth";
const string k_EnableAutoSegmentPropName = "EnableAutoSegment";
public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
{
return GetHeightDrawLayerMasks(property) + 4 * k_LineHeight;
}
public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
{
var indent = EditorGUI.indentLevel;
position.height = k_LineHeight;
EditorGUI.BeginProperty(position, label, property);
var disableCamProp = property.FindPropertyRelative(k_DisableCameraPropName);
EditorGUI.PropertyField(position, disableCamProp, new GUIContent("Disable camera"));
position.y += k_LineHeight;
var enableDepthProp = property.FindPropertyRelative(k_EnableDepthPropName);
EditorGUI.PropertyField(position, enableDepthProp, new GUIContent("Enable depth channel"));
position.y += k_LineHeight;
var enableAutoSegmentProp = property.FindPropertyRelative(k_EnableAutoSegmentPropName);
EditorGUI.PropertyField(position, enableAutoSegmentProp, new GUIContent("Enable auto segment"));
position.y += k_LineHeight;
// Vector Observations
DrawLayerMasks(position, property);
position.y += GetHeightDrawLayerMasks(property);
EditorGUI.EndProperty();
EditorGUI.indentLevel = indent;
}
/// <summary>
/// Draws the Layer Masks for the CameraSensorComponent
/// </summary>
/// <param name="position">Rectangle on the screen to use for the property GUI.</param>
/// <param name="property">The SerializedProperty of the CameraSensorComponent
/// to make the custom GUI for.</param>
static void DrawLayerMasks(Rect position, SerializedProperty property)
{
var layerMasksProp = property.FindPropertyRelative(k_LayerMasksPropName);
var newSize = EditorGUI.IntField(
position,
"# Layer Masks",
layerMasksProp.arraySize
);
// This check is here due to:
// https://fogbugz.unity3d.com/f/cases/1246524/
// If this case has been resolved, please remove this if condition.
if (newSize != layerMasksProp.arraySize)
{
layerMasksProp.arraySize = newSize;
}
position.y += k_LineHeight;
position.x += 20;
position.width -= 20;
for (var branchIndex = 0;
branchIndex < layerMasksProp.arraySize;
branchIndex++)
{
var layerMask =
layerMasksProp.GetArrayElementAtIndex(branchIndex);
var newMaskLayer = EditorGUI.LayerField(
position,
new GUIContent("Mask " + branchIndex + " layer number:",
"Layer number for layer mask " + branchIndex + "."),
layerMask.intValue
);
if (layerMask.intValue != newMaskLayer)
{
layerMask.intValue = newMaskLayer;
}
position.y += k_LineHeight;
}
}
static float GetHeightDrawLayerMasks(SerializedProperty property)
{
var layerMasksProp = property.FindPropertyRelative(k_LayerMasksPropName);
return (1 + layerMasksProp.arraySize) * k_LineHeight;
}
}
}

3
com.unity.ml-agents/Editor/CameraSensorSettingsDrawer.cs.meta


fileFormatVersion: 2
guid: 10f930409af6449fa72f05801f08cfb6
timeCreated: 1592871661

37
com.unity.ml-agents/Runtime/Sensors/CameraSensorSettings.cs


using System;
namespace Unity.MLAgents.Sensors
{
[Serializable]
public class CameraSensorSettings
{
/// <summary>
/// Whether to disable sensor reporting back the raw pixels from the camera.
/// </summary>
/// <remarks>
/// This may be useful if using depth, layer masks, etc. if the raw pixels from the camera
/// would be redundant.
/// </remarks>
public bool DisableCamera;
/// <summary>
/// Whether to add a depth field channel to the sensor.
/// </summary>
public bool EnableDepth;
/// <summary>
/// Whether to enable automatic segmentation by layer
/// </summary>
public bool EnableAutoSegment;
/// <summary>
/// The list of Unity Layers to render masks for.
/// </summary>
/// <remarks>
/// In addition to RGB, Grayscale, or other image channels, we provide "layer masks" as
/// a way to explicitly capture whether objects in a certain Unity Layer fall within each
/// pixel.
/// </remarks>
public int[] LayerMasks = {};
}
}

3
com.unity.ml-agents/Runtime/Sensors/CameraSensorSettings.cs.meta


fileFormatVersion: 2
guid: f02d656dd2124d339fd2056ca0f26839
timeCreated: 1592871589

83
com.unity.ml-agents/Runtime/Sensors/ColorEncoding.cs


using UnityEngine;
namespace Unity.MLAgents.Sensors
{
public class ColorEncoding
{
public static byte ReverseBits(byte value)
{
return (byte)((value * 0x0202020202 & 0x010884422010) % 1023);
}
public static int SparsifyBits(byte value, int sparse)
{
int retVal = 0;
for (int bits = 0; bits < 8; bits++, value >>= 1)
{
retVal |= (value & 1);
retVal <<= sparse;
}
return retVal >> sparse;
}
public static Color EncodeIDAsColor(int instanceId)
{
var uid = instanceId * 2;
if (uid < 0)
uid = -uid + 1;
var sid =
(SparsifyBits((byte)(uid >> 16), 3) << 2) |
(SparsifyBits((byte)(uid >> 8), 3) << 1) |
SparsifyBits((byte)(uid ), 3);
//Debug.Log(uid + " >>> " + System.Convert.ToString(sid, 2).PadLeft(24, '0'));
var r = (byte)(sid >> 8);
var g = (byte)(sid >> 16);
var b = (byte)(sid);
//Debug.Log(r + " " + g + " " + b);
return new Color32 (r, g, b, 255);
}
public static Color EncodeTagAsColor(string tag)
{
var hash = tag.GetHashCode();
var a = (byte)(hash >> 24);
var r = (byte)(hash >> 16);
var g = (byte)(hash >> 8);
var b = (byte)(hash);
return new Color32 (r, g, b, a);
}
public static Color EncodeLayerAsColor(int layer)
{
// Following value must be in the range (0.5 .. 1.0)
// in order to avoid color overlaps when using 'divider' in this func
var z = .7f;
// First 8 layers are Unity Builtin layers
// Unity supports up to 32 layers in total
// Lets create palette of unique 16 colors
var uniqueColors = new []{
new Color(1,1,1,1), new Color(z,z,z,1), // 0
new Color(1,1,z,1), new Color(1,z,1,1), new Color(z,1,1,1), //
new Color(1,z,0,1), new Color(z,0,1,1), new Color(0,1,z,1), // 7
new Color(1,0,0,1), new Color(0,1,0,1), new Color(0,0,1,1), // 8
new Color(1,1,0,1), new Color(1,0,1,1), new Color(0,1,1,1), //
new Color(1,z,z,1), new Color(z,1,z,1) // 15
};
// Create as many colors as necessary by using base 16 color palette
// To create more than 16 - will simply adjust brightness with 'divider'
var color = uniqueColors[layer % uniqueColors.Length];
var divider = 1.0f + Mathf.Floor(layer / uniqueColors.Length);
color /= divider;
return color;
}
}
}

3
com.unity.ml-agents/Runtime/Sensors/ColorEncoding.cs.meta


fileFormatVersion: 2
guid: e3d8e3ee929e4029af6d000dbc7ca686
timeCreated: 1592847947

8
com.unity.ml-agents/Runtime/Shaders.meta


fileFormatVersion: 2
guid: b5af96abaf69d4ed2956e12ef0bf7e8f
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

440
com.unity.ml-agents/Runtime/Shaders/UberReplacement.shader


// Based on builtin Internal-DepthNormalsTexture.shader
// EncodeDepthNormal() is replaced with custom Output() function
Shader "Hidden/UberReplacement" {
Properties {
_MainTex ("", 2D) = "white" {}
_Cutoff ("", Float) = 0.5
_Color ("", Color) = (1,1,1,1)
_ObjectColor ("Object Color", Color) = (1,1,1,1)
_CategoryColor ("Category Color", Color) = (0,1,0,1)
_LayerNumber ("Layer Number", int) = -1
}
SubShader {
CGINCLUDE
fixed4 _ObjectColor;
fixed4 _CategoryColor;
int _LayerNumber;
int _OutputMode;
// remap depth: [0 @ eye .. 1 @ far] => [0 @ near .. 1 @ far]
inline float Linear01FromEyeToLinear01FromNear(float depth01)
{
float near = _ProjectionParams.y;
float far = _ProjectionParams.z;
return (depth01 - near/far) * (1 + near/far);
}
float4 Output(float depth01, float3 normal)
{
if (_OutputMode == 0) // ObjectId
{
return _ObjectColor;
}
else if (_OutputMode == 1) // CategoryId
{
return _CategoryColor;
}
else if (_OutputMode == 2) // DepthCompressed
{
float linearZFromNear = Linear01FromEyeToLinear01FromNear(depth01);
float k = 0.25; // compression factor
return pow(linearZFromNear, k);
}
else if (_OutputMode == 3) // Layers
{
return float4(_LayerNumber / 100.0, 0.0, 0.0, 1);
}
// unsupported _OutputMode
return float4(1, 0.5, 0.5, 1);
}
ENDCG
// Support for different RenderTypes
// The following code is based on builtin Internal-DepthNormalsTexture.shader
Tags { "RenderType"="Opaque" }
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct v2f {
float4 pos : SV_POSITION;
float4 nz : TEXCOORD0;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert( appdata_base v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
o.pos = UnityObjectToClipPos(v.vertex);
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
fixed4 frag(v2f i) : SV_Target {
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TransparentCutout" }
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
uniform float4 _MainTex_ST;
v2f vert( appdata_base v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
uniform fixed4 _Color;
fixed4 frag(v2f i) : SV_Target {
fixed4 texcol = tex2D( _MainTex, i.uv );
clip( texcol.a*_Color.a - _Cutoff );
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TreeBark" }
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "Lighting.cginc"
#include "UnityBuiltin3xTreeLibrary.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert( appdata_full v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TreeVertBark(v);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord.xy;
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
fixed4 frag( v2f i ) : SV_Target {
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TreeLeaf" }
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "Lighting.cginc"
#include "UnityBuiltin3xTreeLibrary.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert( appdata_full v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TreeVertLeaf(v);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord.xy;
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
fixed4 frag( v2f i ) : SV_Target {
half alpha = tex2D(_MainTex, i.uv).a;
clip (alpha - _Cutoff);
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TreeOpaque" "DisableBatching"="True" }
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
float4 nz : TEXCOORD0;
UNITY_VERTEX_OUTPUT_STEREO
};
struct appdata {
float4 vertex : POSITION;
float3 normal : NORMAL;
fixed4 color : COLOR;
UNITY_VERTEX_INPUT_INSTANCE_ID
};
v2f vert( appdata v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TerrainAnimateTree(v.vertex, v.color.w);
o.pos = UnityObjectToClipPos(v.vertex);
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
fixed4 frag(v2f i) : SV_Target {
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TreeTransparentCutout" "DisableBatching"="True" }
Pass {
Cull Back
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
struct appdata {
float4 vertex : POSITION;
float3 normal : NORMAL;
fixed4 color : COLOR;
float4 texcoord : TEXCOORD0;
UNITY_VERTEX_INPUT_INSTANCE_ID
};
v2f vert( appdata v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TerrainAnimateTree(v.vertex, v.color.w);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord.xy;
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
fixed4 frag(v2f i) : SV_Target {
half alpha = tex2D(_MainTex, i.uv).a;
clip (alpha - _Cutoff);
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
Pass {
Cull Front
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
struct appdata {
float4 vertex : POSITION;
float3 normal : NORMAL;
fixed4 color : COLOR;
float4 texcoord : TEXCOORD0;
UNITY_VERTEX_INPUT_INSTANCE_ID
};
v2f vert( appdata v ) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TerrainAnimateTree(v.vertex, v.color.w);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord.xy;
o.nz.xyz = -COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
fixed4 frag(v2f i) : SV_Target {
fixed4 texcol = tex2D( _MainTex, i.uv );
clip( texcol.a - _Cutoff );
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="TreeBillboard" }
Pass {
Cull Off
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert (appdata_tree_billboard v) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
TerrainBillboardTree(v.vertex, v.texcoord1.xy, v.texcoord.y);
o.pos = UnityObjectToClipPos(v.vertex);
o.uv.x = v.texcoord.x;
o.uv.y = v.texcoord.y > 0;
o.nz.xyz = float3(0,0,1);
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
fixed4 frag(v2f i) : SV_Target {
fixed4 texcol = tex2D( _MainTex, i.uv );
clip( texcol.a - 0.001 );
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="GrassBillboard" }
Pass {
Cull Off
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
fixed4 color : COLOR;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert (appdata_full v) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
WavingGrassBillboardVert (v);
o.color = v.color;
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord.xy;
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
fixed4 frag(v2f i) : SV_Target {
fixed4 texcol = tex2D( _MainTex, i.uv );
fixed alpha = texcol.a * i.color.a;
clip( alpha - _Cutoff );
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
SubShader {
Tags { "RenderType"="Grass" }
Pass {
Cull Off
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
#include "TerrainEngine.cginc"
struct v2f {
float4 pos : SV_POSITION;
fixed4 color : COLOR;
float2 uv : TEXCOORD0;
float4 nz : TEXCOORD1;
UNITY_VERTEX_OUTPUT_STEREO
};
v2f vert (appdata_full v) {
v2f o;
UNITY_SETUP_INSTANCE_ID(v);
UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
WavingGrassVert (v);
o.color = v.color;
o.pos = UnityObjectToClipPos(v.vertex);
o.uv = v.texcoord;
o.nz.xyz = COMPUTE_VIEW_NORMAL;
o.nz.w = COMPUTE_DEPTH_01;
return o;
}
uniform sampler2D _MainTex;
uniform fixed _Cutoff;
fixed4 frag(v2f i) : SV_Target {
fixed4 texcol = tex2D( _MainTex, i.uv );
fixed alpha = texcol.a * i.color.a;
clip( alpha - _Cutoff );
return Output (i.nz.w, i.nz.xyz);
}
ENDCG
}
}
Fallback Off
}

9
com.unity.ml-agents/Runtime/Shaders/UberReplacement.shader.meta


fileFormatVersion: 2
guid: 3ffefcbf3b7a34356bd15ac448c73d3e
ShaderImporter:
externalObjects: {}
defaultTextures: []
nonModifiableTextures: []
userData:
assetBundleName:
assetBundleVariant:
正在加载...
取消
保存