Replace JSON.NET with System.Text.json (#840)

src/Artemis.Storage/Artemis.Storage.csproj

@@ -9,6 +9,6 @@
     <ItemGroup>
         <PackageReference Include="LiteDB" />
         <PackageReference Include="Serilog" />
-        <PackageReference Include="Newtonsoft.Json" />
+        <PackageReference Include="System.Text.Json" />
     </ItemGroup>
 </Project>

src/Artemis.Storage/Entities/Profile/AdaptionHints/IAdaptionHintEntity.cs

@@ -1,3 +1,9 @@
-namespace Artemis.Storage.Entities.Profile.AdaptionHints;
+using System.Text.Json.Serialization;
 
+namespace Artemis.Storage.Entities.Profile.AdaptionHints;
+
+[JsonDerivedType(typeof(CategoryAdaptionHintEntity), "Category")]
+[JsonDerivedType(typeof(DeviceAdaptionHintEntity), "Device")]
+[JsonDerivedType(typeof(KeyboardSectionAdaptionHintEntity), "KeyboardSection")]
+[JsonDerivedType(typeof(SingleLedAdaptionHintEntity), "SingleLed")]
 public interface IAdaptionHintEntity;

src/Artemis.Storage/Entities/Profile/Conditions/IConditionEntity.cs

@@ -1,3 +1,10 @@
-namespace Artemis.Storage.Entities.Profile.Abstract;
+using System.Text.Json.Serialization;
+using Artemis.Storage.Entities.Profile.Conditions;
 
+namespace Artemis.Storage.Entities.Profile.Abstract;
+
+[JsonDerivedType(typeof(AlwaysOnConditionEntity), "AlwaysOn")]
+[JsonDerivedType(typeof(EventConditionEntity), "Event")]
+[JsonDerivedType(typeof(PlayOnceConditionEntity), "PlayOnce")]
+[JsonDerivedType(typeof(StaticConditionEntity), "Static")]
 public interface IConditionEntity;

src/Artemis.Storage/Entities/Profile/ProfileConfigurationEntity.cs

@@ -1,5 +1,6 @@
 using System;
 using Artemis.Storage.Entities.Profile.Nodes;
+using Serilog.Core;
 
 namespace Artemis.Storage.Entities.Profile;
 
@@ -26,5 +27,5 @@ public class ProfileConfigurationEntity
     public Guid ProfileId { get; set; }
 
     public bool FadeInAndOut { get; set; }
-    public int Version { get; set; }
+    public int Version { get; set; } = StorageMigrationService.PROFILE_VERSION;
 }

src/Artemis.Storage/Migrations/IProfileMigration.cs

@@ -1,9 +1,9 @@
-using Newtonsoft.Json.Linq;
+using System.Text.Json.Nodes;
 
 namespace Artemis.Storage.Migrations;
 
 public interface IProfileMigration
 {
     int Version { get; }
-    void Migrate(JObject configurationJson, JObject profileJson);
+    void Migrate(JsonObject configurationJson, JsonObject profileJson);
 }

src/Artemis.Storage/Migrations/Profile/M0001NodeProviders.cs

@@ -1,88 +1,91 @@
-using Newtonsoft.Json.Linq;
+using System.Text.Json.Nodes;
 
-namespace Artemis.Storage.Migrations.Profile;
-
-/// <summary>
-/// Migrates nodes to be provider-based.
-/// This requires giving them a ProviderId and updating the their namespaces to match the namespace of the new plugin. 
-/// </summary>
-internal class M0001NodeProviders : IProfileMigration
+namespace Artemis.Storage.Migrations.Profile
 {
-    /// <inheritdoc />
-    public int Version => 1;
-
-    /// <inheritdoc />
-    public void Migrate(JObject configurationJson, JObject profileJson)
+    /// <summary>
+    /// Migrates nodes to be provider-based.
+    /// This requires giving them a ProviderId and updating the their namespaces to match the namespace of the new plugin. 
+    /// </summary>
+    internal class M0001NodeProviders : IProfileMigration
     {
-        JArray? folders = (JArray?) profileJson["Folders"]?["$values"];
-        JArray? layers = (JArray?) profileJson["Layers"]?["$values"];
+        /// <inheritdoc />
+        public int Version => 1;
 
-        if (folders != null)
+        /// <inheritdoc />
+        public void Migrate(JsonObject configurationJson, JsonObject profileJson)
         {
-            foreach (JToken folder in folders)
-                MigrateProfileElement(folder);
+            JsonArray? folders = profileJson["Folders"]?["$values"]?.AsArray();
+            JsonArray? layers = profileJson["Layers"]?["$values"]?.AsArray();
+
+            if (folders != null)
+            {
+                foreach (JsonNode? folder in folders)
+                    MigrateProfileElement(folder);
+            }
+
+            if (layers != null)
+            {
+                foreach (JsonNode? layer in layers)
+                {
+                    MigrateProfileElement(layer);
+                    MigratePropertyGroup(layer?["GeneralPropertyGroup"]);
+                    MigratePropertyGroup(layer?["TransformPropertyGroup"]);
+                    MigratePropertyGroup(layer?["LayerBrush"]?["PropertyGroup"]);
+                }
+            }
         }
 
-        if (layers != null)
+        private void MigrateProfileElement(JsonNode? profileElement)
         {
-            foreach (JToken layer in layers)
+            if (profileElement == null)
+                return;
+            
+            JsonArray? layerEffects = profileElement["LayerEffects"]?["$values"]?.AsArray();
+            if (layerEffects != null)
             {
-                MigrateProfileElement(layer);
-                MigratePropertyGroup(layer["GeneralPropertyGroup"]);
-                MigratePropertyGroup(layer["TransformPropertyGroup"]);
-                MigratePropertyGroup(layer["LayerBrush"]?["PropertyGroup"]);
+                foreach (JsonNode? layerEffect in layerEffects)
+                    MigratePropertyGroup(layerEffect?["PropertyGroup"]);
+            }
+
+            JsonNode? displayCondition = profileElement["DisplayCondition"];
+            if (displayCondition != null)
+                MigrateNodeScript(displayCondition["Script"]);
+        }
+
+        private void MigratePropertyGroup(JsonNode? propertyGroup)
+        {
+            if (propertyGroup == null)
+                return;
+
+            JsonArray? properties = propertyGroup["Properties"]?["$values"]?.AsArray();
+            JsonArray? propertyGroups = propertyGroup["PropertyGroups"]?["$values"]?.AsArray();
+            if (properties != null)
+            {
+                foreach (JsonNode? property in properties)
+                    MigrateNodeScript(property?["DataBinding"]?["NodeScript"]);
+            }
+
+            if (propertyGroups != null)
+            {
+                foreach (JsonNode? childPropertyGroup in propertyGroups)
+                    MigratePropertyGroup(childPropertyGroup);
+            }
+        }
+
+        private void MigrateNodeScript(JsonNode? nodeScript)
+        {
+            JsonArray? nodes = nodeScript?["Nodes"]?["$values"]?.AsArray();
+            if (nodes == null)
+                return;
+
+            foreach (JsonNode? jsonNode in nodes)
+            {
+                if (jsonNode == null)
+                    continue;
+                JsonObject nodeObject = jsonNode.AsObject();
+                nodeObject["Type"] = nodeObject["Type"]?.GetValue<string>().Replace("Artemis.VisualScripting.Nodes", "Artemis.Plugins.Nodes.General.Nodes");
+                nodeObject["ProviderId"] = "Artemis.Plugins.Nodes.General.GeneralNodesProvider-d9e1ee78";
             }
         }
     }
-
-    private void MigrateProfileElement(JToken profileElement)
-    {
-        JArray? layerEffects = (JArray?) profileElement["LayerEffects"]?["$values"];
-        if (layerEffects != null)
-        {
-            foreach (JToken layerEffect in layerEffects)
-                MigratePropertyGroup(layerEffect["PropertyGroup"]);
-        }
-
-        JToken? displayCondition = profileElement["DisplayCondition"];
-        if (displayCondition != null)
-            MigrateNodeScript(displayCondition["Script"]);
-    }
-
-    private void MigratePropertyGroup(JToken? propertyGroup)
-    {
-        if (propertyGroup == null || !propertyGroup.HasValues)
-            return;
-
-        JArray? properties = (JArray?) propertyGroup["Properties"]?["$values"];
-        JArray? propertyGroups = (JArray?) propertyGroup["PropertyGroups"]?["$values"];
-
-        if (properties != null)
-        {
-            foreach (JToken property in properties)
-                MigrateNodeScript(property["DataBinding"]?["NodeScript"]);
-        }
-
-        if (propertyGroups != null)
-        {
-            foreach (JToken childPropertyGroup in propertyGroups)
-                MigratePropertyGroup(childPropertyGroup);
-        }
-    }
-
-    private void MigrateNodeScript(JToken? nodeScript)
-    {
-        if (nodeScript == null || !nodeScript.HasValues)
-            return;
-
-        JArray? nodes = (JArray?) nodeScript["Nodes"]?["$values"];
-        if (nodes == null)
-            return;
-
-        foreach (JToken node in nodes)
-        {
-            node["Type"] = node["Type"]?.Value<string>()?.Replace("Artemis.VisualScripting.Nodes", "Artemis.Plugins.Nodes.General.Nodes");
-            node["ProviderId"] = "Artemis.Plugins.Nodes.General.GeneralNodesProvider-d9e1ee78";
-        }
-    }
-}
+}

src/Artemis.Storage/Migrations/Profile/M0002NodeProvidersProfileConfig.cs

@@ -1,4 +1,4 @@
-using Newtonsoft.Json.Linq;
+using System.Text.Json.Nodes;
 
 namespace Artemis.Storage.Migrations.Profile;
 
@@ -12,24 +12,24 @@ internal class M0002NodeProvidersProfileConfig : IProfileMigration
     public int Version => 2;
 
     /// <inheritdoc />
-    public void Migrate(JObject configurationJson, JObject profileJson)
+    public void Migrate(JsonObject configurationJson, JsonObject profileJson)
     {
         MigrateNodeScript(configurationJson["ActivationCondition"]);
     }
-    
-    private void MigrateNodeScript(JToken? nodeScript)
-    {
-        if (nodeScript == null || !nodeScript.HasValues)
-            return;
 
-        JArray? nodes = (JArray?) nodeScript["Nodes"]?["$values"];
+    private void MigrateNodeScript(JsonNode? nodeScript)
+    {
+        JsonArray? nodes = nodeScript?["Nodes"]?["$values"]?.AsArray();
         if (nodes == null)
             return;
 
-        foreach (JToken node in nodes)
+        foreach (JsonNode? jsonNode in nodes)
         {
-            node["Type"] = node["Type"]?.Value<string>()?.Replace("Artemis.VisualScripting.Nodes", "Artemis.Plugins.Nodes.General.Nodes");
-            node["ProviderId"] = "Artemis.Plugins.Nodes.General.GeneralNodesProvider-d9e1ee78";
+            if (jsonNode == null)
+                continue;
+            JsonObject nodeObject = jsonNode.AsObject();
+            nodeObject["Type"] = nodeObject["Type"]?.GetValue<string>().Replace("Artemis.VisualScripting.Nodes", "Artemis.Plugins.Nodes.General.Nodes");
+            nodeObject["ProviderId"] = "Artemis.Plugins.Nodes.General.GeneralNodesProvider-d9e1ee78";
         }
     }
 }

src/Artemis.Storage/Migrations/Profile/M0003SystemTextJson.cs

@@ -0,0 +1,84 @@
+using System.Linq;
+using System.Text.Json.Nodes;
+
+namespace Artemis.Storage.Migrations.Profile;
+
+/// <summary>
+/// Migrates profiles to be deserializable by System.Text.Json, removing type information from the JSON arrays and most objects.
+/// </summary>
+internal class M0003SystemTextJson : IProfileMigration
+{
+    /// <inheritdoc />
+    public int Version => 3;
+
+    /// <inheritdoc />
+    public void Migrate(JsonObject configurationJson, JsonObject profileJson)
+    {
+        ConvertToSystemTextJson(configurationJson);
+        ConvertToSystemTextJson(profileJson);
+    }
+
+    private void ConvertToSystemTextJson(JsonObject jsonObject)
+    {
+        FilterType(jsonObject);
+        
+        // Recursively convert all JSON arrays from {$type: "xyz", $values: []} to []
+        foreach ((string? key, JsonNode? value) in jsonObject.ToDictionary())
+        {
+            if (value is not JsonObject obj)
+                continue;
+
+            // if there is a $type and a $values, replace the entire node with $values regardless of the value of $type
+            if (obj["$type"] != null && obj["$values"] != null)
+            {
+                JsonArray? values = obj["$values"]?.AsArray();
+                if (values != null)
+                {
+                    obj.Remove("$values");
+                    jsonObject[key] = values;
+                    foreach (JsonNode? jsonNode in values.ToList())
+                    {
+                        if (jsonNode is JsonObject childObject)
+                            ConvertToSystemTextJson(childObject);
+                    }
+                }
+
+                obj.Remove("$type");
+            }
+            else
+            {
+                ConvertToSystemTextJson(obj);
+            }
+        }
+    }
+
+    private void FilterType(JsonObject jsonObject)
+    {
+        // Replace or remove $type depending on whether there's a matching JsonDerivedType
+        // This could be done with reflection but that would mean this migration automatically gains new behaviour over time.
+        JsonNode? type = jsonObject["$type"];
+        if (type != null)
+        {
+            // Adaption Hints
+            if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.AdaptionHints.CategoryAdaptionHintEntity, Artemis.Storage")
+                jsonObject["$type"] = "Category";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.AdaptionHints.DeviceAdaptionHintEntity, Artemis.Storage")
+                jsonObject["$type"] = "Device";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.AdaptionHints.KeyboardSectionAdaptionHintEntity, Artemis.Storage")
+                jsonObject["$type"] = "KeyboardSection";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.AdaptionHints.SingleLedAdaptionHintEntity, Artemis.Storage")
+                jsonObject["$type"] = "SingleLed";
+            // Conditions
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.Conditions.AlwaysOnConditionEntity, Artemis.Storage")
+                jsonObject["$type"] = "AlwaysOn";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.Conditions.EventConditionEntity, Artemis.Storage")
+                jsonObject["$type"] = "Event";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.Conditions.PlayOnceConditionEntity, Artemis.Storage")
+                jsonObject["$type"] = "PlayOnce";
+            else if (type.GetValue<string>() == "Artemis.Storage.Entities.Profile.Conditions.StaticConditionEntity, Artemis.Storage")
+                jsonObject["$type"] = "Static";
+            else
+                jsonObject.Remove("$type");
+        }
+    }
+}

src/Artemis.Storage/Repositories/ProfileCategoryRepository.cs

@@ -31,12 +31,24 @@ internal class ProfileCategoryRepository : IProfileCategoryRepository
 
     public List<ProfileCategoryEntity> GetAll()
     {
-        return _repository.Query<ProfileCategoryEntity>().ToList();
+        List<ProfileCategoryEntity> categories = _repository.Query<ProfileCategoryEntity>().ToList();
+        
+        // Update all profile versions to the current version, profile migrations don't apply to LiteDB so anything loadable is assumed to be up to date
+        foreach (ProfileCategoryEntity profileCategoryEntity in categories)
+            UpdateProfileVersions(profileCategoryEntity);
+
+        return categories;
     }
 
     public ProfileCategoryEntity? Get(Guid id)
     {
-        return _repository.FirstOrDefault<ProfileCategoryEntity>(p => p.Id == id);
+        ProfileCategoryEntity? result = _repository.FirstOrDefault<ProfileCategoryEntity>(p => p.Id == id);
+        if (result == null)
+            return null;
+        
+        // Update all profile versions to the current version, profile migrations don't apply to LiteDB so anything loadable is assumed to be up to date
+        UpdateProfileVersions(result);
+        return result;
     }
 
     public ProfileCategoryEntity IsUnique(string name, Guid? id)
@@ -72,4 +84,10 @@ internal class ProfileCategoryRepository : IProfileCategoryRepository
 
         _profileIcons.Upload(profileConfigurationEntity.FileIconId, profileConfigurationEntity.FileIconId + ".png", stream);
     }
+    
+    private static void UpdateProfileVersions(ProfileCategoryEntity profileCategoryEntity)
+    {
+        foreach (ProfileConfigurationEntity profileConfigurationEntity in profileCategoryEntity.ProfileConfigurations)
+            profileConfigurationEntity.Version = StorageMigrationService.PROFILE_VERSION;
+    }
 }

src/Artemis.Storage/StorageMigrationService.cs

@@ -9,6 +9,8 @@ namespace Artemis.Storage;
 
 public class StorageMigrationService
 {
+    public const int PROFILE_VERSION = 3;
+    
     private readonly ILogger _logger;
     private readonly IList<IStorageMigration> _migrations;
     private readonly LiteRepository _repository;