Upgrade deps (#68)

* Upgrading projects and dependencies

* Adding additional editor versions for testing.

* Adding test ensuring datasets are generated on exiting play mode.

* Reducing test set for per-commit to a single editor. Removing use of IncludeTestAssemblies from BuildPlayer* tests to work around mysterious issue with shader compilation.

* Fix yaml

* Adding "All Tests" which triggers nightly

* Switching to newer 2019.4

* Fixing instance and semantic segmentation shaders in 2020.1. Bumping package version. Removing publish tests from all tests and renaming

* Promotion test rename

* Update to latest capture and core. Switch off 2020.2 testing because it is not currently supported. Switch CI back to 2019.4.6f1 due to linux editor import hang.

* Updating changelog

.yamato/environments.yml

 
 upmci_registry: https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
 
-test_editors:
+# sticking to 2019.4.6f1 for testing for now because Linux Editor fails to open PerceptionHDRP on 2019.4.8f1
+# see https://fogbugz.unity3d.com/default.asp?1273518_d68j5lb6eucglb84
+coverage_editors:
+  - version: 2019.4.6f1
+
+per_commit_editors:
-publish_editors:
+complete_editors:
+  - version: 2020.1.3f1
+#  - version: 2020.2.0a21
 
 publish_platforms:
   - name: win

.yamato/promote.yml

 
 ---
 {% for variant in package_variants %}
-{% for editor in test_editors %}
+{% for editor in complete_editors %}
-  name : Package tests ({{variant.name}} pkg, {{ editor.version }}, {{ platform.name }})
+  name : Package promotion tests ({{variant.name}} pkg, {{ editor.version }}, {{ platform.name }})
   agent:
     type: {{ platform.type }}
     image: {{ platform.image }}
 promotion_test_trigger:
   name: Promotion Tests Trigger
   dependencies:
-    {% for editor in publish_editors %}
+    {% for editor in complete_editors %}
     {% for platform in publish_platforms %}
     {% for suite in suites %}
     {% for project in projects %}
         - "upm-ci~/packages/*.tgz"
   dependencies:
     {% for variant in package_variants %}
-    {% for editor in publish_editors %}
+    {% for editor in complete_editors %}
     {% for platform in publish_platforms %}
     - .yamato/upm-ci-full.yml#pack_{{ variant.name }}
     - .yamato/promote.yml#promotion_test_{{ platform.name }}_{{ editor.version }}

.yamato/publish.yml

         - "upm-ci~/packages/*.tgz"
   dependencies:
     - .yamato/upm-ci-full.yml#pack_Perception
-    {% for editor in publish_editors %}
+    {% for editor in complete_editors %}
     {% for platform in publish_platforms %}
     {% for suite in suites %}
     {% for project in projects %}

.yamato/upm-ci-full.yml

 {% endfor %}
 
 {% for variant in package_variants %}
-{% for editor in test_editors %}
+{% for editor in complete_editors %}
 {% for platform in test_platforms %}
 pkg_test_{{variant.name}}_{{ platform.name }}_{{ editor.version }}:
   name : Package tests ({{variant.name}} pkg, {{ editor.version }}, {{ platform.name }})
 {% endfor %}
 
 test_trigger:
-  name: All Tests
+  name: Per Commit Tests
   agent:
     type: Unity::VM
     image: package-ci/win10:stable
     branches:
       only:
         - "/.*/"
+    cancel_old_ci: true
+  artifacts:
+    logs:
+      paths:
+        - "upm-ci~/test-results/**/*"
+    packages:
+      paths:
+        - "upm-ci~/packages/**/*"
+  dependencies:
+    {% for variant in package_variants %}
+    {% for editor in per_commit_editors %}
+    {% for platform in test_platforms %}
+    - .yamato/upm-ci-full.yml#pkg_test_{{variant.name}}_{{platform.name}}_{{editor.version}}
+    {% endfor %}
+    {% endfor %}
+    {% endfor %}
+
+    {% for editor in coverage_editors %}
+    - .yamato/upm-ci-testprojects.yml#codecoverage_windows_editmode_{{editor.version}}
+    {% endfor %}
+    {% for editor in per_commit_editors %}
+    {% for project in projects %}
+    - .yamato/upm-ci-testprojects.yml#{{project.name}}_windows_standalone_{{editor.version}}
+    - .yamato/upm-ci-testprojects.yml#{{project.name}}_linux_standalone_{{editor.version}}
+    - .yamato/upm-ci-testprojects.yml#{{project.name}}_windows_editmode_{{editor.version}}
+    - .yamato/upm-ci-testprojects.yml#{{project.name}}_linux_editmode_{{editor.version}}
+    {% endfor %}
+    {% endfor %}
+
+all_tests:
+  name: Complete Tests
+  agent:
+    type: Unity::VM
+    image: package-ci/win10:stable
+    flavor: b1.small
+  commands:
+    - dir
+  triggers:
     recurring:
       - branch: master
         frequency: daily # Should run between midnight and 6AM UTC
         - "upm-ci~/packages/**/*"
   dependencies:
     {% for variant in package_variants %}
-    {% for editor in test_editors %}
+    {% for editor in complete_editors %}
     {% for platform in test_platforms %}
     - .yamato/upm-ci-full.yml#pkg_test_{{variant.name}}_{{platform.name}}_{{editor.version}}
     {% endfor %}
-    # Disabling trigger of standalone tests which aren't working at the moment
-    {% for editor in test_editors %}
+    {% for editor in coverage_editors %}
+    {% endfor %}
+    {% for editor in complete_editors %}
     {% for project in projects %}
     - .yamato/upm-ci-testprojects.yml#{{project.name}}_windows_standalone_{{editor.version}}
     - .yamato/upm-ci-testprojects.yml#{{project.name}}_linux_standalone_{{editor.version}}

.yamato/upm-ci-testprojects.yml

 
 ---
 
-{% for editor in test_editors %}
+{% for editor in complete_editors %}
 {% for suite in suites %}
 {% for project in projects %}
 {{project.name}}_windows_{{suite.name}}_{{editor.version}}:
 {% endfor %}
 {% endfor %}
 
-{% for editor in test_editors %}
+{% for editor in coverage_editors %}
 {% for suite in suites %}
 codecoverage_windows_{{suite.name}}_{{editor.version}}:
   name : CodeCoverage {{ suite.display_name }} tests ({{ editor.version }}, Windows)
 {% endfor %}
 {% endfor %}
 
-{% for editor in test_editors %}
+{% for editor in complete_editors %}
 {% for suite in suites %}
 {% for project in projects %}
 {{project.name}}_linux_{{suite.name}}_{{editor.version}}:
 # Not including OSX because the only OSX platform on Bokken with a GPU is a Mac Mini, of which there are only a few and setting up Yamato jobs is very complicated.
 
 # {% for variant in package_variants %}
-# {% for editor in test_editors %}
+# {% for editor in complete_editors %}
 # code_coverage_win_{{editor.version}}:
 #   name: Code Coverage Report - Windows
 #   agent:

TestProjects/PerceptionHDRP/Packages/manifest.json

     "com.unity.perception": "file:../../../com.unity.perception",
     "com.unity.render-pipelines.core": "7.3.1",
     "com.unity.render-pipelines.high-definition": "7.3.1",
-    "com.unity.simulation.capture": "0.0.10-preview.10",
-    "com.unity.simulation.core": "0.0.10-preview.17",
-    "com.unity.test-framework": "1.1.14",
+    "com.unity.simulation.capture": "0.0.10-preview.12",
+    "com.unity.simulation.core": "0.0.10-preview.19",
+    "com.unity.test-framework": "1.1.16",
     "com.unity.testtools.codecoverage": "0.2.2-preview",
     "com.unity.textmeshpro": "2.0.1",
     "com.unity.ugui": "1.0.0",

TestProjects/PerceptionHDRP/Packages/packages-lock.json

         "com.unity.nuget.newtonsoft-json": "1.1.2",
         "com.unity.render-pipelines.core": "7.1.6",
         "com.unity.entities": "0.8.0-preview.8",
-        "com.unity.simulation.capture": "0.0.10-preview.10",
-        "com.unity.simulation.core": "0.0.10-preview.17"
+        "com.unity.simulation.capture": "0.0.10-preview.12",
+        "com.unity.simulation.core": "0.0.10-preview.19"
       }
     },
     "com.unity.platforms": {
       "url": "https://packages.unity.com"
     },
     "com.unity.simulation.capture": {
-      "version": "0.0.10-preview.10",
+      "version": "0.0.10-preview.12",
-        "com.unity.simulation.core": "0.0.10-preview.17"
+        "com.unity.simulation.core": "0.0.10-preview.19"
-      "version": "0.0.10-preview.17",
+      "version": "0.0.10-preview.19",
       "depth": 0,
       "source": "registry",
       "dependencies": {},
-      "version": "1.1.14",
+      "version": "1.1.16",
       "depth": 0,
       "source": "registry",
       "dependencies": {

TestProjects/PerceptionHDRP/ProjectSettings/ProjectSettings.asset

   useOnDemandResources: 0
   accelerometerFrequency: 60
   companyName: DefaultCompany
-  productName: UnityTestFramework
+  productName: PerceptionHDRP
   defaultCursor: {fileID: 0}
   cursorHotspot: {x: 0, y: 0}
   m_SplashScreenBackgroundColor: {r: 0.13725491, g: 0.12156863, b: 0.1254902, a: 1}
   ps4ShareFilePath: 
   ps4ShareOverlayImagePath: 
   ps4PrivacyGuardImagePath: 
+  ps4ExtraSceSysFile: 
   ps4NPtitleDatPath: 
   ps4RemotePlayKeyAssignment: -1
   ps4RemotePlayKeyMappingDir: 
   ps4disableAutoHideSplash: 0
   ps4videoRecordingFeaturesUsed: 0
   ps4contentSearchFeaturesUsed: 0
+  ps4CompatibilityPS5: 0
+  ps4GPU800MHz: 1
   ps4attribEyeToEyeDistanceSettingVR: 0
   ps4IncludedModules: []
   ps4attribVROutputEnabled: 0

TestProjects/PerceptionHDRP/ProjectSettings/ProjectVersion.txt

-m_EditorVersion: 2019.4.7f1
-m_EditorVersionWithRevision: 2019.4.7f1 (e992b1a16e65)
+m_EditorVersion: 2019.4.8f1
+m_EditorVersionWithRevision: 2019.4.8f1 (60781d942082)

TestProjects/PerceptionURP/Packages/manifest.json

     "com.unity.package-validation-suite": "0.9.1-preview",
     "com.unity.perception": "file:../../../com.unity.perception",
     "com.unity.render-pipelines.universal": "7.3.1",
-    "com.unity.simulation.capture": "0.0.10-preview.10",
-    "com.unity.simulation.core": "0.0.10-preview.17",
-    "com.unity.test-framework": "1.1.14",
+    "com.unity.simulation.capture": "0.0.10-preview.12",
+    "com.unity.simulation.core": "0.0.10-preview.19",
+    "com.unity.test-framework": "1.1.16",
     "com.unity.textmeshpro": "2.0.1",
     "com.unity.timeline": "1.2.12",
     "com.unity.ugui": "1.0.0",

TestProjects/PerceptionURP/Packages/packages-lock.json

         "com.unity.nuget.newtonsoft-json": "1.1.2",
         "com.unity.render-pipelines.core": "7.1.6",
         "com.unity.entities": "0.8.0-preview.8",
-        "com.unity.simulation.capture": "0.0.10-preview.10",
-        "com.unity.simulation.core": "0.0.10-preview.17"
+        "com.unity.simulation.capture": "0.0.10-preview.12",
+        "com.unity.simulation.core": "0.0.10-preview.19"
       }
     },
     "com.unity.platforms": {
       "url": "https://packages.unity.com"
     },
     "com.unity.simulation.capture": {
-      "version": "0.0.10-preview.10",
+      "version": "file:com.unity.simulation.capture",
-      "source": "registry",
+      "source": "embedded",
-        "com.unity.simulation.core": "0.0.10-preview.17"
-      },
-      "url": "https://packages.unity.com"
+        "com.unity.simulation.core": "0.0.10-preview.19"
+      }
-      "version": "0.0.10-preview.17",
+      "version": "0.0.10-preview.19",
       "depth": 0,
       "source": "registry",
       "dependencies": {},
-      "version": "1.1.14",
+      "version": "1.1.16",
       "depth": 0,
       "source": "registry",
       "dependencies": {

TestProjects/PerceptionURP/ProjectSettings/ProjectSettings.asset

   switchNVNShaderPoolsGranularity: 33554432
   switchNVNDefaultPoolsGranularity: 16777216
   switchNVNOtherPoolsGranularity: 16777216
+  switchNVNMaxPublicTextureIDCount: 0
+  switchNVNMaxPublicSamplerIDCount: 0
+  stadiaPresentMode: 0
+  stadiaTargetFramerate: 0
+  vulkanEnableLateAcquireNextImage: 0
   m_SupportedAspectRatios:
     4:3: 1
     5:4: 1
   uIStatusBarHidden: 1
   uIExitOnSuspend: 0
   uIStatusBarStyle: 0
-  iPhoneSplashScreen: {fileID: 0}
-  iPhoneHighResSplashScreen: {fileID: 0}
-  iPhoneTallHighResSplashScreen: {fileID: 0}
-  iPhone47inSplashScreen: {fileID: 0}
-  iPhone55inPortraitSplashScreen: {fileID: 0}
-  iPhone55inLandscapeSplashScreen: {fileID: 0}
-  iPhone58inPortraitSplashScreen: {fileID: 0}
-  iPhone58inLandscapeSplashScreen: {fileID: 0}
-  iPadPortraitSplashScreen: {fileID: 0}
-  iPadHighResPortraitSplashScreen: {fileID: 0}
-  iPadLandscapeSplashScreen: {fileID: 0}
-  iPadHighResLandscapeSplashScreen: {fileID: 0}
-  iPhone65inPortraitSplashScreen: {fileID: 0}
-  iPhone65inLandscapeSplashScreen: {fileID: 0}
-  iPhone61inPortraitSplashScreen: {fileID: 0}
-  iPhone61inLandscapeSplashScreen: {fileID: 0}
   appleTVSplashScreen: {fileID: 0}
   appleTVSplashScreen2x: {fileID: 0}
   tvOSSmallIconLayers: []
   - m_BuildTarget: WebGLSupport
     m_APIs: 0b000000
     m_Automatic: 0
+  - m_BuildTarget: WindowsStandaloneSupport
+    m_APIs: 02000000
+    m_Automatic: 0
   m_BuildTargetVRSettings: []
   openGLRequireES31: 0
   openGLRequireES31AEP: 0
   ps4ShareFilePath: 
   ps4ShareOverlayImagePath: 
   ps4PrivacyGuardImagePath: 
+  ps4ExtraSceSysFile: 
   ps4NPtitleDatPath: 
   ps4RemotePlayKeyAssignment: -1
   ps4RemotePlayKeyMappingDir: 
   ps4UseResolutionFallback: 0
   ps4ReprojectionSupport: 0
   ps4UseAudio3dBackend: 0
+  ps4UseLowGarlicFragmentationMode: 1
   ps4SocialScreenEnabled: 0
   ps4ScriptOptimizationLevel: 0
   ps4Audio3dVirtualSpeakerCount: 14
   ps4disableAutoHideSplash: 0
   ps4videoRecordingFeaturesUsed: 0
   ps4contentSearchFeaturesUsed: 0
+  ps4CompatibilityPS5: 0
+  ps4GPU800MHz: 1
   ps4attribEyeToEyeDistanceSettingVR: 0
   ps4IncludedModules: []
   ps4attribVROutputEnabled: 0
   XboxOnePersistentLocalStorageSize: 0
   XboxOneXTitleMemory: 8
   XboxOneOverrideIdentityName: 
+  XboxOneOverrideIdentityPublisher: 
   vrEditorSettings:
     daydream:
       daydreamIconForeground: {fileID: 0}

TestProjects/PerceptionURP/ProjectSettings/ProjectVersion.txt

-m_EditorVersion: 2019.4.7f1
-m_EditorVersionWithRevision: 2019.4.7f1 (e992b1a16e65)
+m_EditorVersion: 2019.4.8f1
+m_EditorVersionWithRevision: 2019.4.8f1 (60781d942082)

com.unity.perception/CHANGELOG.md

 ### Added
 Added new randomization tools
 
+Added support for 2020.1
+
-Changed default labeler ids to be lower-case to be consistent with the ids in the dataset.
+Changed default labeler ids to be lower-case to be consistent with the ids in the dataset
+
+Switched to latest versions of com.unity.simulation.core and com.unity.simulation.capture
 
 ### Deprecated
 

com.unity.perception/Documentation~/index.md

 |[Perception Camera](PerceptionCamera.md)|Captures RGB images and ground truth from a [Camera](https://docs.unity3d.com/Manual/class-Camera.html)|
 |[DatasetCapture](DatasetCapture.md)|Ensures sensors are triggered at proper rates and accepts data for the JSON dataset|
 |[Randomization](Randomization/Index.md)|Integrate domain randomization principles into your simulation|
+
+## Known Issues
+
+* The Linux Editor 2019.4.7f1 and 2019.4.8f1 have been found to hang when importing HDRP-based perception projects. For Linux Editor support, use 2019.4.6f1 or 2020.1

com.unity.perception/Runtime/GroundTruth/Resources/InstanceSegmentation.shader

             #pragma vertex vert
             #pragma fragment frag
 
-            #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Common.hlsl"
-            #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Packing.hlsl"
+            #include "UnityCG.cginc"
+            #include "Packing.hlsl"
 
             struct appdata
             {

com.unity.perception/Runtime/GroundTruth/Resources/SemanticSegmentation.shader

             #pragma vertex vert
             #pragma fragment frag
 
-            #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Common.hlsl"
-            #include "Packages/com.unity.render-pipelines.core/ShaderLibrary/Packing.hlsl"
+            #include "UnityCG.cginc"
 
             float4 LabelingId;
 

com.unity.perception/Tests/Editor/BuildPerceptionPlayer.cs

         [Test]
         public void BuildPlayerStandaloneWindows64()
         {
-            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneWindows64, m_BuildPath, BuildOptions.IncludeTestAssemblies, out _, out m_Summary);
+            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneWindows64, m_BuildPath, BuildOptions.None, out _, out m_Summary);
             Assert.AreEqual(BuildResult.Succeeded, m_Summary.result, " BuildTarget.StandaloneWindows64 failed to build");
         }
 
         {
-            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneLinux64, m_BuildPath, BuildOptions.IncludeTestAssemblies, out _, out m_Summary);
+            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneLinux64, m_BuildPath, BuildOptions.None, out _, out m_Summary);
             Assert.AreEqual(BuildResult.Succeeded, m_Summary.result, "BuildTarget.StandaloneLinux64 failed to build");
         }
 
         public void BuildPlayerOSX()
         {
-            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneOSX, m_BuildPath, BuildOptions.IncludeTestAssemblies, out _, out m_Summary);
+            BuildPlayer(BuildTargetGroup.Standalone, BuildTarget.StandaloneOSX, m_BuildPath, BuildOptions.None, out _, out m_Summary);
             Assert.AreEqual(BuildResult.Succeeded, m_Summary.result, "BuildTarget.StandaloneLinux64 failed to build");
         }
 

com.unity.perception/Tests/Editor/DatasetCaptureEditorTests.cs

 using System;
+using System.Collections;
+using System.IO;
+using UnityEngine;
+using UnityEngine.TestTools;
+    [TestFixture]
+    [Serializable]
+        [SerializeField]
+        string expectedDatasetPath;
         [Test]
         public void RegisterEgo_InEditMode_Throws()
         {
         public void RegisterMetricDefinition_InEditMode_Throws()
         {
             Assert.Throws<InvalidOperationException>(() => DatasetCapture.RegisterMetricDefinition(""));
+        }
+        [UnityTest]
+        public IEnumerator SimpleData_GeneratesFullDataset_OnExitPlaymode()
+        {
+            yield return new EnterPlayMode();
+            var ego = DatasetCapture.RegisterEgo("ego");
+            var sensor = DatasetCapture.RegisterSensor(ego, "camera", "", 0.1f, 0);
+            sensor.ReportCapture("file.txt", new SensorSpatialData());
+            expectedDatasetPath = DatasetCapture.OutputDirectory;
+            yield return new ExitPlayMode();
+            FileAssert.Exists(Path.Combine(expectedDatasetPath, "sensors.json"));
         }
     }
 }

com.unity.perception/package.json

     "com.unity.nuget.newtonsoft-json": "1.1.2",
     "com.unity.render-pipelines.core": "7.1.6",
     "com.unity.entities": "0.8.0-preview.8",
-    "com.unity.simulation.capture": "0.0.10-preview.10",
-    "com.unity.simulation.core": "0.0.10-preview.17"
+    "com.unity.simulation.capture": "0.0.10-preview.12",
+    "com.unity.simulation.core": "0.0.10-preview.19"
-  "version": "0.3.0-preview.1"
+  "version": "0.4.0-preview.1"
 }

com.unity.perception/Runtime/GroundTruth/Resources/Packing.hlsl

+#ifndef UNITY_PERCEPTION_PACKING_INCLUDED
+#define UNITY_PERCEPTION_PACKING_INCLUDED
+
+#define real float
+#define real2 float2
+#define real3 float3
+#define real4 float4
+
+// Packs an integer stored using at most 'numBits' into a [0..1] real.
+real PackInt(uint i, uint numBits)
+{
+    uint maxInt = (1u << numBits) - 1u;
+    return saturate(i * rcp(maxInt));
+}
+
+// Unpacks a [0..1] real into an integer of size 'numBits'.
+uint UnpackInt(real f, uint numBits)
+{
+    uint maxInt = (1u << numBits) - 1u;
+    return (uint)(f * maxInt + 0.5); // Round instead of truncating
+}
+
+#ifndef INTRINSIC_BITFIELD_EXTRACT
+// Unsigned integer bit field extraction.
+// Note that the intrinsic itself generates a vector instruction.
+// Wrap this function with WaveReadLaneFirst() to get scalar output.
+uint BitFieldExtract(uint data, uint offset, uint numBits)
+{
+    uint mask = (1u << numBits) - 1u;
+    return (data >> offset) & mask;
+}
+#endif // INTRINSIC_BITFIELD_EXTRACT
+
+//-----------------------------------------------------------------------------
+// Float packing
+//-----------------------------------------------------------------------------
+
+// src must be between 0.0 and 1.0
+uint PackFloatToUInt(real src, uint offset, uint numBits)
+{
+    return UnpackInt(src, numBits) << offset;
+}
+
+real UnpackUIntToFloat(uint src, uint offset, uint numBits)
+{
+    uint maxInt = (1u << numBits) - 1u;
+    return real(BitFieldExtract(src, offset, numBits)) * rcp(maxInt);
+}
+
+#endif // UNITY_PACKING_INCLUDED

com.unity.perception/Runtime/GroundTruth/Resources/Packing.hlsl.meta

+fileFormatVersion: 2
+guid: 9af0f66010764cc5a8ef4acccc164cdb
+timeCreated: 1598539851