Yaml stuff (#496)

* Move converters into separate namespace

* Improve yaml logic, add system for discriminated unions

This should be a lot cleaner and safer, witout any breaking changes.

* Some cleanup

---------

Co-authored-by: cognite-bulldozer[bot] <51074376+cognite-bulldozer[bot]@users.noreply.github.com>

Cognite.Config/KeyVault.cs

@@ -97,22 +97,6 @@ public SecretClient GetClient()
             );
             return _client;
         }
-
-        /// <summary>
-        /// Add a key vault node deserializer and tag mapping to a yaml deserializer builder.
-        /// </summary>
-        /// <param name="builder">Builder to add key vault support to</param>
-        internal void AddKeyVault(DeserializerBuilder builder)
-        {
-            if (builder == null) throw new ArgumentNullException(nameof(builder));
-            try
-            {
-                builder.WithoutNodeDeserializer(typeof(KeyVaultResolver));
-            }
-            catch { }
-
-            builder.WithNodeDeserializer(new KeyVaultResolver(this));
-        }
     }
 
     internal class KeyVaultResolver : INodeDeserializer

Cognite.Config/Yaml/DefaultFilterTypeInspector.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+using YamlDotNet.Serialization.TypeInspectors;
+
+namespace Cognite.Extractor.Configuration
+{
+    /// <summary>
+    /// YamlDotNet type inspector, used to filter out default values from the generated config.
+    /// Instead of serializing the entire config file, which ends up being complicated and difficult to read,
+    /// this just serializes the properties that do not simply equal the default values.
+    /// This does sometimes produce empty arrays, but we can strip those later.
+    /// </summary>
+    internal class DefaultFilterTypeInspector : TypeInspectorSkeleton
+    {
+        private readonly ITypeInspector _innerTypeDescriptor;
+        private readonly HashSet<string> _toAlwaysKeep;
+        private readonly HashSet<string> _toIgnore;
+        private readonly IEnumerable<string> _namePrefixes;
+        private readonly IEnumerable<IYamlTypeConverter> _customConverters;
+        private readonly bool _allowReadOnly;
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="innerTypeDescriptor">Inner type descriptor</param>
+        /// <param name="toAlwaysKeep">Fields to always keep</param>
+        /// <param name="toIgnore">Fields to exclude</param>
+        /// <param name="namePrefixes">Prefixes on full type names for types that should be explored internally</param>
+        /// <param name="customConverters">List of registered custom converters.</param>
+        /// <param name="allowReadOnly">Allow read only properties</param>
+        public DefaultFilterTypeInspector(
+            ITypeInspector innerTypeDescriptor,
+            IEnumerable<string> toAlwaysKeep,
+            IEnumerable<string> toIgnore,
+            IEnumerable<string> namePrefixes,
+            IEnumerable<IYamlTypeConverter> customConverters,
+            bool allowReadOnly)
+        {
+            _innerTypeDescriptor = innerTypeDescriptor;
+            _toAlwaysKeep = new HashSet<string>(toAlwaysKeep);
+            _toIgnore = new HashSet<string>(toIgnore);
+            _namePrefixes = namePrefixes;
+            _allowReadOnly = allowReadOnly;
+            _customConverters = customConverters;
+        }
+
+        /// <inheritdoc />
+        public override IEnumerable<IPropertyDescriptor> GetProperties(Type type, object? container)
+        {
+            if (container is null || type is null) return Enumerable.Empty<IPropertyDescriptor>();
+            var props = _innerTypeDescriptor.GetProperties(type, container);
+
+            object? dfs = null;
+            try
+            {
+                dfs = Activator.CreateInstance(type);
+                var genD = type.GetMethod("GenerateDefaults");
+                genD?.Invoke(dfs, null);
+            }
+            catch { }
+
+            props = props.Where(p =>
+            {
+                var name = PascalCaseNamingConvention.Instance.Apply(p.Name);
+
+                // Some config objects have private properties, since this is a write-back of config we shouldn't save those
+                if (!p.CanWrite && !_allowReadOnly) return false;
+                // Some custom properties are kept on the config object for convenience
+                if (_toIgnore.Contains(name)) return false;
+                // Some should be kept to encourage users to set them
+                if (_toAlwaysKeep.Contains(name)) return true;
+
+                var prop = type.GetProperty(name);
+                object? df = null;
+                if (dfs != null) df = prop?.GetValue(dfs);
+                var val = prop?.GetValue(container);
+
+                if (val != null && prop != null && !type.IsValueType
+                    && _namePrefixes.Any(prefix => prop.PropertyType.FullName!.StartsWith(prefix, StringComparison.InvariantCulture))
+                    // Any type covered by a custom converter shouldn't be passed through here. We don't know
+                    // how those are serialized, it is likely not just by listing their properties.
+                    && _customConverters.All(conv => !conv.Accepts(prop.PropertyType)))
+                {
+                    var pr = GetProperties(prop.PropertyType, val);
+                    if (!pr.Any()) return false;
+                }
+
+
+                // No need to emit empty lists.
+                if (val != null && (val is IEnumerable list) && !list.GetEnumerator().MoveNext()) return false;
+
+                // Compare the value of each property with its default, and check for empty arrays, don't save those.
+                // This creates minimal config files
+                return df != null && !df.Equals(val) || df == null && val != null;
+            });
+
+            return props;
+        }
+    }
+}

Cognite.Config/Yaml/ListOrStringConverter.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Cognite.Common;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Serialization;
+
+namespace Cognite.Extractor.Configuration
+{
+    internal class ListOrStringConverter : IYamlTypeConverter
+    {
+        private static readonly Regex _whitespaceRegex = new Regex(@"\s", RegexOptions.Compiled | RegexOptions.CultureInvariant);
+        public bool Accepts(Type type)
+        {
+            return type == typeof(ListOrSpaceSeparated);
+        }
+
+        public object? ReadYaml(IParser parser, Type type)
+        {
+            if (parser.TryConsume<Scalar>(out var scalar))
+            {
+                var items = _whitespaceRegex.Split(scalar.Value);
+                return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray());
+            }
+            if (parser.TryConsume<SequenceStart>(out _))
+            {
+                var items = new List<string>();
+                while (!parser.Accept<SequenceEnd>(out _))
+                {
+                    var seqScalar = parser.Consume<Scalar>();
+                    items.Add(seqScalar.Value);
+                }
+
+                parser.Consume<SequenceEnd>();
+
+                return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray());
+            }
+
+            throw new InvalidOperationException("Expected list or value");
+        }
+
+        public void WriteYaml(IEmitter emitter, object? value, Type type)
+        {
+            emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true,
+                SequenceStyle.Block, Mark.Empty, Mark.Empty));
+            var it = value as ListOrSpaceSeparated;
+            foreach (var elem in it!.Values)
+            {
+                emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, elem, ScalarStyle.DoubleQuoted, false, true));
+            }
+            emitter.Emit(new SequenceEnd());
+        }
+    }
+}

Cognite.Config/Yaml/TemplatedValueDeserializer.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Text.RegularExpressions;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Serialization;
+
+namespace Cognite.Extractor.Configuration
+{
+    internal class TemplatedValueDeserializer : INodeDeserializer
+    {
+        public TemplatedValueDeserializer()
+        {
+        }
+
+        private static bool IsNumericType(Type t)
+        {
+            var tc = Type.GetTypeCode(t);
+            return tc >= TypeCode.SByte && tc <= TypeCode.Decimal;
+        }
+
+        private static readonly Regex _envRegex = new Regex(@"\$\{([A-Za-z0-9_]+)\}", RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+        bool INodeDeserializer.Deserialize(IParser parser, Type expectedType, Func<IParser, Type, object?> nestedObjectDeserializer, out object? value)
+        {
+            if (expectedType != typeof(string) && !IsNumericType(expectedType))
+            {
+                value = null;
+                return false;
+            }
+
+            if (parser.Accept<Scalar>(out var scalar) && scalar != null && _envRegex.IsMatch(scalar.Value))
+            {
+                parser.MoveNext();
+                value = Replace(scalar.Value);
+                return true;
+            }
+            value = null;
+            return false;
+        }
+
+        public static string Replace(string toReplace)
+        {
+            return _envRegex.Replace(toReplace, LookupEnvironment);
+        }
+
+        private static string LookupEnvironment(Match match)
+        {
+            return Environment.GetEnvironmentVariable(match.Groups[1].Value) ?? "";
+        }
+    }
+}

Cognite.Config/Yaml/YamlEnumConverter.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.Serialization;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Serialization;
+
+namespace Cognite.Extractor.Configuration
+{
+    internal class YamlEnumConverter : IYamlTypeConverter
+    {
+        public bool Accepts(Type type)
+        {
+            return type.IsEnum || (Nullable.GetUnderlyingType(type)?.IsEnum ?? false);
+        }
+
+        public object? ReadYaml(IParser parser, Type type)
+        {
+            var scalar = parser.Consume<Scalar>();
+
+            if (scalar.Value == null)
+            {
+                if (Nullable.GetUnderlyingType(type) != null)
+                {
+                    return null;
+                }
+                throw new YamlException($"Failed to deserialize null value to enum {type.Name}");
+            }
+
+            type = Nullable.GetUnderlyingType(type) ?? type;
+
+            var values = type.GetMembers()
+                .Select(m => (m.GetCustomAttributes<EnumMemberAttribute>(true).Select(f => f.Value).FirstOrDefault(), m))
+                .Where(pair => !string.IsNullOrEmpty(pair.Item1))
+                .ToDictionary(pair => pair.Item1!, pair => pair.m);
+
+            if (values.TryGetValue(scalar.Value, out var enumMember))
+            {
+                return Enum.Parse(type, enumMember.Name, true);
+            }
+            return Enum.Parse(type, scalar.Value, true);
+        }
+
+        public void WriteYaml(IEmitter emitter, object? value, Type type)
+        {
+            if (value == null) return;
+            var member = type.GetMember(value.ToString() ?? "").FirstOrDefault();
+            var stringValue = member?.GetCustomAttributes<EnumMemberAttribute>(true)?.Select(f => f.Value)?.FirstOrDefault() ?? value.ToString();
+            emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, stringValue!, ScalarStyle.DoubleQuoted, false, true));
+        }
+    }
+}