From 21f08c8bdadbe3d7dbae215cf510504ad7dcebfe Mon Sep 17 00:00:00 2001 From: Einar Date: Thu, 5 Dec 2024 11:04:45 +0100 Subject: [PATCH] Yaml stuff (#496) * Move converters into separate namespace * Improve yaml logic, add system for discriminated unions This should be a lot cleaner and safer, witout any breaking changes. * Some cleanup --------- Co-authored-by: cognite-bulldozer[bot] <51074376+cognite-bulldozer[bot]@users.noreply.github.com> --- Cognite.Config/Configuration.cs | 381 ++---------------- Cognite.Config/KeyVault.cs | 16 - .../Yaml/DefaultFilterTypeInspector.cs | 103 +++++ Cognite.Config/Yaml/ListOrStringConverter.cs | 56 +++ .../Yaml/TemplatedValueDeserializer.cs | 51 +++ Cognite.Config/Yaml/YamlEnumConverter.cs | 53 +++ Cognite.Config/YamlConfigBuilder.cs | 283 +++++++++++++ ExtractorUtils.Test/unit/ConfigurationTest.cs | 41 ++ 8 files changed, 611 insertions(+), 373 deletions(-) create mode 100644 Cognite.Config/Yaml/DefaultFilterTypeInspector.cs create mode 100644 Cognite.Config/Yaml/ListOrStringConverter.cs create mode 100644 Cognite.Config/Yaml/TemplatedValueDeserializer.cs create mode 100644 Cognite.Config/Yaml/YamlEnumConverter.cs create mode 100644 Cognite.Config/YamlConfigBuilder.cs diff --git a/Cognite.Config/Configuration.cs b/Cognite.Config/Configuration.cs index df3fbf82..800c349f 100644 --- a/Cognite.Config/Configuration.cs +++ b/Cognite.Config/Configuration.cs @@ -1,22 +1,14 @@ using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Linq.Expressions; using System.Reflection; -using System.Runtime.Serialization; using System.Text.RegularExpressions; -using Cognite.Common; using Cognite.Extractor.Common; using Cognite.Extractor.KeyVault; using Microsoft.Extensions.DependencyInjection; using YamlDotNet.Core; -using YamlDotNet.Core.Events; using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.TypeInspectors; -using YamlDotNet.Serialization.Utilities; namespace Cognite.Extractor.Configuration { @@ -27,32 +19,7 @@ namespace Cognite.Extractor.Configuration /// public static class ConfigurationUtils { - private static DeserializerBuilder builder = new DeserializerBuilder() - .WithNamingConvention(HyphenatedNamingConvention.Instance) - .WithTypeConverter(new ListOrStringConverter()) - .WithTagMapping("!keyvault", typeof(object)) - .WithNodeDeserializer(new TemplatedValueDeserializer()) - .WithTypeConverter(new YamlEnumConverter()); - private static IDeserializer deserializer = builder - .Build(); - private static DeserializerBuilder ignoreUnmatchedBuilder = new DeserializerBuilder() - .WithNamingConvention(HyphenatedNamingConvention.Instance) - .WithNodeDeserializer(new TemplatedValueDeserializer()) - .WithTagMapping("!keyvault", typeof(object)) - .WithTypeConverter(new ListOrStringConverter()) - .WithTypeConverter(new YamlEnumConverter()) - .IgnoreUnmatchedProperties(); - private static IDeserializer ignoreUnmatchedDeserializer = ignoreUnmatchedBuilder.Build(); - private static IDeserializer failOnUnmatchedDeserializer = builder.Build(); - - private static readonly List converters = new List() { - new ListOrStringConverter(), - new YamlEnumConverter(), - }; - - private static bool ignoreUnmatchedProperties; - - private static object _deserializerLock = new object(); + private static YamlConfigBuilder _builder = new YamlConfigBuilder(); /// /// Reads the provided string containing yml and deserializes it to an object of type . @@ -67,10 +34,8 @@ public static T ReadString(string yaml, bool ignoreUnmatched = false) { try { - lock (_deserializerLock) - { - return GetDeserializer(ignoreUnmatched).Deserialize(yaml); - } + _builder.IgnoreUnmatchedProperties = ignoreUnmatched; + return _builder.Deserializer.Deserialize(yaml); } catch (YamlException ye) { @@ -78,13 +43,6 @@ public static T ReadString(string yaml, bool ignoreUnmatched = false) } } - private static IDeserializer GetDeserializer(bool? ignoreUnmatched) - { - if (ignoreUnmatched == null) return deserializer; - - return ignoreUnmatched.Value ? ignoreUnmatchedDeserializer : failOnUnmatchedDeserializer; - } - /// /// Reads the yaml file found in the provided path and deserializes it to an object of type /// Values containing ${ENV_VARIABLE} will be replaced by the environment variable of the same name. @@ -101,10 +59,8 @@ public static T Read(string path, bool? ignoreUnmatched = null) { using (var reader = File.OpenText(path)) { - lock (_deserializerLock) - { - return GetDeserializer(ignoreUnmatched).Deserialize(reader); - } + _builder.IgnoreUnmatchedProperties = ignoreUnmatched ?? false; + return _builder.Deserializer.Deserialize(reader); } } catch (System.IO.FileNotFoundException fnfe) @@ -235,21 +191,6 @@ public static T TryReadConfigFromFile(string path, bool ignoreUnmatched, para return config; } - private static void Rebuild() - { - ignoreUnmatchedDeserializer = ignoreUnmatchedBuilder.Build(); - failOnUnmatchedDeserializer = builder.Build(); - - if (ignoreUnmatchedProperties) - { - deserializer = ignoreUnmatchedDeserializer; - } - else - { - deserializer = failOnUnmatchedDeserializer; - } - } - /// /// Maps the given tag to the type T. /// Mapping is only required for custom tags. @@ -258,22 +199,7 @@ private static void Rebuild() /// Type to map to public static void AddTagMapping(string tag) { - lock (_deserializerLock) - { - try - { - builder.WithoutTagMapping(tag); - } - catch { } - builder = builder.WithTagMapping(tag, typeof(T)); - try - { - ignoreUnmatchedBuilder.WithoutTagMapping(tag); - } - catch { } - ignoreUnmatchedBuilder = ignoreUnmatchedBuilder.WithTagMapping(tag, typeof(T)); - Rebuild(); - } + _builder.AddTagMapping(tag); } /// @@ -282,25 +208,20 @@ public static void AddTagMapping(string tag) /// Type converter to add public static void AddTypeConverter(IYamlTypeConverter converter) { - if (converter == null) throw new ArgumentNullException(nameof(converter)); - lock (_deserializerLock) - { - try - { - builder.WithoutTypeConverter(converter.GetType()); - } - catch { } - builder = builder.WithTypeConverter(converter); - try - { - ignoreUnmatchedBuilder.WithoutTypeConverter(converter.GetType()); - } - catch { } - ignoreUnmatchedBuilder = ignoreUnmatchedBuilder.WithTypeConverter(converter); - converters.RemoveAll(conv => converter.GetType() == conv.GetType()); - converters.Add(converter); - Rebuild(); - } + _builder.AddTypeConverter(converter); + } + + /// + /// Add an internally tagged type to the yaml deserializer. + /// + /// The type in your actual config structure that + /// indicates that this is a custom mapping. + /// + /// The key for the discriminator, i.e. "type" + /// A map from discriminator key value to type + public static void AddDiscriminatedType(string key, IDictionary variants) + { + _builder.AddDiscriminatedType(key, variants); } /// @@ -308,11 +229,7 @@ public static void AddTypeConverter(IYamlTypeConverter converter) /// public static void IgnoreUnmatchedProperties() { - ignoreUnmatchedProperties = true; - lock (_deserializerLock) - { - Rebuild(); - } + _builder.IgnoreUnmatchedProperties = true; } /// @@ -320,11 +237,7 @@ public static void IgnoreUnmatchedProperties() /// public static void DisallowUnmatchedProperties() { - ignoreUnmatchedProperties = false; - lock (_deserializerLock) - { - Rebuild(); - } + _builder.IgnoreUnmatchedProperties = false; } /// @@ -333,14 +246,7 @@ public static void DisallowUnmatchedProperties() /// public static void AddKeyVault(KeyVaultConfig config) { - if (config == null) throw new ArgumentNullException(nameof(config)); - - lock (_deserializerLock) - { - config.AddKeyVault(builder); - config.AddKeyVault(ignoreUnmatchedBuilder); - Rebuild(); - } + _builder.AddKeyVault(config); } [System.Diagnostics.CodeAnalysis.SuppressMessage("Maintainability", "CA1508: Avoid dead conditional code", Justification = "Other methods using this can still pass null as parameter")] @@ -431,22 +337,7 @@ public static string ConfigToString( { if (config is null) return ""; - var builder = new SerializerBuilder() - .WithTypeInspector(insp => new DefaultFilterTypeInspector( - insp, - toAlwaysKeep, - toIgnore, - namePrefixes, - converters, - allowReadOnly)) - .WithNamingConvention(HyphenatedNamingConvention.Instance); - - foreach (var converter in converters) - { - builder.WithTypeConverter(converter); - } - - var serializer = builder.Build(); + var serializer = _builder.GetSafeSerializer(toAlwaysKeep, toIgnore, namePrefixes, allowReadOnly); string raw = serializer.Serialize(config); @@ -471,228 +362,4 @@ public static string TrimConfigString(string raw) return raw; } } - - internal class TemplatedValueDeserializer : INodeDeserializer - { - public TemplatedValueDeserializer() - { - } - - private static bool IsNumericType(Type t) - { - var tc = Type.GetTypeCode(t); - return tc >= TypeCode.SByte && tc <= TypeCode.Decimal; - } - - private static readonly Regex _envRegex = new Regex(@"\$\{([A-Za-z0-9_]+)\}", RegexOptions.Compiled | RegexOptions.CultureInvariant); - - bool INodeDeserializer.Deserialize(IParser parser, Type expectedType, Func nestedObjectDeserializer, out object? value) - { - if (expectedType != typeof(string) && !IsNumericType(expectedType)) - { - value = null; - return false; - } - - if (parser.Accept(out var scalar) && scalar != null && _envRegex.IsMatch(scalar.Value)) - { - parser.MoveNext(); - value = Replace(scalar.Value); - return true; - } - value = null; - return false; - } - - public static string Replace(string toReplace) - { - return _envRegex.Replace(toReplace, LookupEnvironment); - } - - private static string LookupEnvironment(Match match) - { - return Environment.GetEnvironmentVariable(match.Groups[1].Value) ?? ""; - } - } - - /// - /// YamlDotNet type inspector, used to filter out default values from the generated config. - /// Instead of serializing the entire config file, which ends up being complicated and difficult to read, - /// this just serializes the properties that do not simply equal the default values. - /// This does sometimes produce empty arrays, but we can strip those later. - /// - internal class DefaultFilterTypeInspector : TypeInspectorSkeleton - { - private readonly ITypeInspector _innerTypeDescriptor; - private readonly HashSet _toAlwaysKeep; - private readonly HashSet _toIgnore; - private readonly IEnumerable _namePrefixes; - private readonly IEnumerable _customConverters; - private readonly bool _allowReadOnly; - /// - /// Constructor - /// - /// Inner type descriptor - /// Fields to always keep - /// Fields to exclude - /// Prefixes on full type names for types that should be explored internally - /// List of registered custom converters. - /// Allow read only properties - public DefaultFilterTypeInspector( - ITypeInspector innerTypeDescriptor, - IEnumerable toAlwaysKeep, - IEnumerable toIgnore, - IEnumerable namePrefixes, - IEnumerable customConverters, - bool allowReadOnly) - { - _innerTypeDescriptor = innerTypeDescriptor; - _toAlwaysKeep = new HashSet(toAlwaysKeep); - _toIgnore = new HashSet(toIgnore); - _namePrefixes = namePrefixes; - _allowReadOnly = allowReadOnly; - _customConverters = customConverters; - } - - /// - public override IEnumerable GetProperties(Type type, object? container) - { - if (container is null || type is null) return Enumerable.Empty(); - var props = _innerTypeDescriptor.GetProperties(type, container); - - object? dfs = null; - try - { - dfs = Activator.CreateInstance(type); - var genD = type.GetMethod("GenerateDefaults"); - genD?.Invoke(dfs, null); - } - catch { } - - props = props.Where(p => - { - var name = PascalCaseNamingConvention.Instance.Apply(p.Name); - - // Some config objects have private properties, since this is a write-back of config we shouldn't save those - if (!p.CanWrite && !_allowReadOnly) return false; - // Some custom properties are kept on the config object for convenience - if (_toIgnore.Contains(name)) return false; - // Some should be kept to encourage users to set them - if (_toAlwaysKeep.Contains(name)) return true; - - var prop = type.GetProperty(name); - object? df = null; - if (dfs != null) df = prop?.GetValue(dfs); - var val = prop?.GetValue(container); - - if (val != null && prop != null && !type.IsValueType - && _namePrefixes.Any(prefix => prop.PropertyType.FullName!.StartsWith(prefix, StringComparison.InvariantCulture)) - // Any type covered by a custom converter shouldn't be passed through here. We don't know - // how those are serialized, it is likely not just by listing their properties. - && _customConverters.All(conv => !conv.Accepts(prop.PropertyType))) - { - var pr = GetProperties(prop.PropertyType, val); - if (!pr.Any()) return false; - } - - - // No need to emit empty lists. - if (val != null && (val is IEnumerable list) && !list.GetEnumerator().MoveNext()) return false; - - // Compare the value of each property with its default, and check for empty arrays, don't save those. - // This creates minimal config files - return df != null && !df.Equals(val) || df == null && val != null; - }); - - return props; - } - } - - internal class ListOrStringConverter : IYamlTypeConverter - { - private static readonly Regex _whitespaceRegex = new Regex(@"\s", RegexOptions.Compiled | RegexOptions.CultureInvariant); - public bool Accepts(Type type) - { - return type == typeof(ListOrSpaceSeparated); - } - - public object? ReadYaml(IParser parser, Type type) - { - if (parser.TryConsume(out var scalar)) - { - var items = _whitespaceRegex.Split(scalar.Value); - return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); - } - if (parser.TryConsume(out _)) - { - var items = new List(); - while (!parser.Accept(out _)) - { - var seqScalar = parser.Consume(); - items.Add(seqScalar.Value); - } - - parser.Consume(); - - return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); - } - - throw new InvalidOperationException("Expected list or value"); - } - - public void WriteYaml(IEmitter emitter, object? value, Type type) - { - emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true, - SequenceStyle.Block, Mark.Empty, Mark.Empty)); - var it = value as ListOrSpaceSeparated; - foreach (var elem in it!.Values) - { - emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, elem, ScalarStyle.DoubleQuoted, false, true)); - } - emitter.Emit(new SequenceEnd()); - } - } - - internal class YamlEnumConverter : IYamlTypeConverter - { - public bool Accepts(Type type) - { - return type.IsEnum || (Nullable.GetUnderlyingType(type)?.IsEnum ?? false); - } - - public object? ReadYaml(IParser parser, Type type) - { - var scalar = parser.Consume(); - - if (scalar.Value == null) - { - if (Nullable.GetUnderlyingType(type) != null) - { - return null; - } - throw new YamlException($"Failed to deserialize null value to enum {type.Name}"); - } - - type = Nullable.GetUnderlyingType(type) ?? type; - - var values = type.GetMembers() - .Select(m => (m.GetCustomAttributes(true).Select(f => f.Value).FirstOrDefault(), m)) - .Where(pair => !string.IsNullOrEmpty(pair.Item1)) - .ToDictionary(pair => pair.Item1!, pair => pair.m); - - if (values.TryGetValue(scalar.Value, out var enumMember)) - { - return Enum.Parse(type, enumMember.Name, true); - } - return Enum.Parse(type, scalar.Value, true); - } - - public void WriteYaml(IEmitter emitter, object? value, Type type) - { - if (value == null) return; - var member = type.GetMember(value.ToString() ?? "").FirstOrDefault(); - var stringValue = member?.GetCustomAttributes(true)?.Select(f => f.Value)?.FirstOrDefault() ?? value.ToString(); - emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, stringValue!, ScalarStyle.DoubleQuoted, false, true)); - } - } } diff --git a/Cognite.Config/KeyVault.cs b/Cognite.Config/KeyVault.cs index 92238285..a3c147b9 100644 --- a/Cognite.Config/KeyVault.cs +++ b/Cognite.Config/KeyVault.cs @@ -97,22 +97,6 @@ public SecretClient GetClient() ); return _client; } - - /// - /// Add a key vault node deserializer and tag mapping to a yaml deserializer builder. - /// - /// Builder to add key vault support to - internal void AddKeyVault(DeserializerBuilder builder) - { - if (builder == null) throw new ArgumentNullException(nameof(builder)); - try - { - builder.WithoutNodeDeserializer(typeof(KeyVaultResolver)); - } - catch { } - - builder.WithNodeDeserializer(new KeyVaultResolver(this)); - } } internal class KeyVaultResolver : INodeDeserializer diff --git a/Cognite.Config/Yaml/DefaultFilterTypeInspector.cs b/Cognite.Config/Yaml/DefaultFilterTypeInspector.cs new file mode 100644 index 00000000..24473012 --- /dev/null +++ b/Cognite.Config/Yaml/DefaultFilterTypeInspector.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.TypeInspectors; + +namespace Cognite.Extractor.Configuration +{ + /// + /// YamlDotNet type inspector, used to filter out default values from the generated config. + /// Instead of serializing the entire config file, which ends up being complicated and difficult to read, + /// this just serializes the properties that do not simply equal the default values. + /// This does sometimes produce empty arrays, but we can strip those later. + /// + internal class DefaultFilterTypeInspector : TypeInspectorSkeleton + { + private readonly ITypeInspector _innerTypeDescriptor; + private readonly HashSet _toAlwaysKeep; + private readonly HashSet _toIgnore; + private readonly IEnumerable _namePrefixes; + private readonly IEnumerable _customConverters; + private readonly bool _allowReadOnly; + /// + /// Constructor + /// + /// Inner type descriptor + /// Fields to always keep + /// Fields to exclude + /// Prefixes on full type names for types that should be explored internally + /// List of registered custom converters. + /// Allow read only properties + public DefaultFilterTypeInspector( + ITypeInspector innerTypeDescriptor, + IEnumerable toAlwaysKeep, + IEnumerable toIgnore, + IEnumerable namePrefixes, + IEnumerable customConverters, + bool allowReadOnly) + { + _innerTypeDescriptor = innerTypeDescriptor; + _toAlwaysKeep = new HashSet(toAlwaysKeep); + _toIgnore = new HashSet(toIgnore); + _namePrefixes = namePrefixes; + _allowReadOnly = allowReadOnly; + _customConverters = customConverters; + } + + /// + public override IEnumerable GetProperties(Type type, object? container) + { + if (container is null || type is null) return Enumerable.Empty(); + var props = _innerTypeDescriptor.GetProperties(type, container); + + object? dfs = null; + try + { + dfs = Activator.CreateInstance(type); + var genD = type.GetMethod("GenerateDefaults"); + genD?.Invoke(dfs, null); + } + catch { } + + props = props.Where(p => + { + var name = PascalCaseNamingConvention.Instance.Apply(p.Name); + + // Some config objects have private properties, since this is a write-back of config we shouldn't save those + if (!p.CanWrite && !_allowReadOnly) return false; + // Some custom properties are kept on the config object for convenience + if (_toIgnore.Contains(name)) return false; + // Some should be kept to encourage users to set them + if (_toAlwaysKeep.Contains(name)) return true; + + var prop = type.GetProperty(name); + object? df = null; + if (dfs != null) df = prop?.GetValue(dfs); + var val = prop?.GetValue(container); + + if (val != null && prop != null && !type.IsValueType + && _namePrefixes.Any(prefix => prop.PropertyType.FullName!.StartsWith(prefix, StringComparison.InvariantCulture)) + // Any type covered by a custom converter shouldn't be passed through here. We don't know + // how those are serialized, it is likely not just by listing their properties. + && _customConverters.All(conv => !conv.Accepts(prop.PropertyType))) + { + var pr = GetProperties(prop.PropertyType, val); + if (!pr.Any()) return false; + } + + + // No need to emit empty lists. + if (val != null && (val is IEnumerable list) && !list.GetEnumerator().MoveNext()) return false; + + // Compare the value of each property with its default, and check for empty arrays, don't save those. + // This creates minimal config files + return df != null && !df.Equals(val) || df == null && val != null; + }); + + return props; + } + } +} diff --git a/Cognite.Config/Yaml/ListOrStringConverter.cs b/Cognite.Config/Yaml/ListOrStringConverter.cs new file mode 100644 index 00000000..375332cc --- /dev/null +++ b/Cognite.Config/Yaml/ListOrStringConverter.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Cognite.Common; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace Cognite.Extractor.Configuration +{ + internal class ListOrStringConverter : IYamlTypeConverter + { + private static readonly Regex _whitespaceRegex = new Regex(@"\s", RegexOptions.Compiled | RegexOptions.CultureInvariant); + public bool Accepts(Type type) + { + return type == typeof(ListOrSpaceSeparated); + } + + public object? ReadYaml(IParser parser, Type type) + { + if (parser.TryConsume(out var scalar)) + { + var items = _whitespaceRegex.Split(scalar.Value); + return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); + } + if (parser.TryConsume(out _)) + { + var items = new List(); + while (!parser.Accept(out _)) + { + var seqScalar = parser.Consume(); + items.Add(seqScalar.Value); + } + + parser.Consume(); + + return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); + } + + throw new InvalidOperationException("Expected list or value"); + } + + public void WriteYaml(IEmitter emitter, object? value, Type type) + { + emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true, + SequenceStyle.Block, Mark.Empty, Mark.Empty)); + var it = value as ListOrSpaceSeparated; + foreach (var elem in it!.Values) + { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, elem, ScalarStyle.DoubleQuoted, false, true)); + } + emitter.Emit(new SequenceEnd()); + } + } +} diff --git a/Cognite.Config/Yaml/TemplatedValueDeserializer.cs b/Cognite.Config/Yaml/TemplatedValueDeserializer.cs new file mode 100644 index 00000000..f018ab91 --- /dev/null +++ b/Cognite.Config/Yaml/TemplatedValueDeserializer.cs @@ -0,0 +1,51 @@ +using System; +using System.Text.RegularExpressions; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace Cognite.Extractor.Configuration +{ + internal class TemplatedValueDeserializer : INodeDeserializer + { + public TemplatedValueDeserializer() + { + } + + private static bool IsNumericType(Type t) + { + var tc = Type.GetTypeCode(t); + return tc >= TypeCode.SByte && tc <= TypeCode.Decimal; + } + + private static readonly Regex _envRegex = new Regex(@"\$\{([A-Za-z0-9_]+)\}", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + bool INodeDeserializer.Deserialize(IParser parser, Type expectedType, Func nestedObjectDeserializer, out object? value) + { + if (expectedType != typeof(string) && !IsNumericType(expectedType)) + { + value = null; + return false; + } + + if (parser.Accept(out var scalar) && scalar != null && _envRegex.IsMatch(scalar.Value)) + { + parser.MoveNext(); + value = Replace(scalar.Value); + return true; + } + value = null; + return false; + } + + public static string Replace(string toReplace) + { + return _envRegex.Replace(toReplace, LookupEnvironment); + } + + private static string LookupEnvironment(Match match) + { + return Environment.GetEnvironmentVariable(match.Groups[1].Value) ?? ""; + } + } +} diff --git a/Cognite.Config/Yaml/YamlEnumConverter.cs b/Cognite.Config/Yaml/YamlEnumConverter.cs new file mode 100644 index 00000000..24db492a --- /dev/null +++ b/Cognite.Config/Yaml/YamlEnumConverter.cs @@ -0,0 +1,53 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.Serialization; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace Cognite.Extractor.Configuration +{ + internal class YamlEnumConverter : IYamlTypeConverter + { + public bool Accepts(Type type) + { + return type.IsEnum || (Nullable.GetUnderlyingType(type)?.IsEnum ?? false); + } + + public object? ReadYaml(IParser parser, Type type) + { + var scalar = parser.Consume(); + + if (scalar.Value == null) + { + if (Nullable.GetUnderlyingType(type) != null) + { + return null; + } + throw new YamlException($"Failed to deserialize null value to enum {type.Name}"); + } + + type = Nullable.GetUnderlyingType(type) ?? type; + + var values = type.GetMembers() + .Select(m => (m.GetCustomAttributes(true).Select(f => f.Value).FirstOrDefault(), m)) + .Where(pair => !string.IsNullOrEmpty(pair.Item1)) + .ToDictionary(pair => pair.Item1!, pair => pair.m); + + if (values.TryGetValue(scalar.Value, out var enumMember)) + { + return Enum.Parse(type, enumMember.Name, true); + } + return Enum.Parse(type, scalar.Value, true); + } + + public void WriteYaml(IEmitter emitter, object? value, Type type) + { + if (value == null) return; + var member = type.GetMember(value.ToString() ?? "").FirstOrDefault(); + var stringValue = member?.GetCustomAttributes(true)?.Select(f => f.Value)?.FirstOrDefault() ?? value.ToString(); + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, stringValue!, ScalarStyle.DoubleQuoted, false, true)); + } + } +} diff --git a/Cognite.Config/YamlConfigBuilder.cs b/Cognite.Config/YamlConfigBuilder.cs new file mode 100644 index 00000000..8a544200 --- /dev/null +++ b/Cognite.Config/YamlConfigBuilder.cs @@ -0,0 +1,283 @@ +using System; +using System.Collections.Generic; +using Cognite.Extractor.KeyVault; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Cognite.Extractor.Configuration +{ + internal class DiscriminatedUnionConfig + { + public Type BaseType { get; } + public string Key { get; } + public IDictionary Variants { get; } + + public DiscriminatedUnionConfig(Type baseType, string key, IDictionary variants) + { + BaseType = baseType; + Key = key; + Variants = variants; + } + } + + /// + /// Custom builder for YamlDotNet deserializers. + /// + /// This handles idempotency better, so that it can be used in a static context. + /// + public class YamlConfigBuilder + { + private IDeserializer? _deserializer; + + /// + /// Get a deserializer with the current config. + /// + /// This will only rebuild if the config has changed. + /// + public IDeserializer Deserializer + { + get + { + lock (_changeLock) + { + if (_deserializer == null || _changed) + { + _deserializer = Build(); + } + _changed = false; + return _deserializer; + } + } + } + + + private bool _changed; + private object _changeLock = new object(); + + private INamingConvention _namingConvention = HyphenatedNamingConvention.Instance; + + /// + /// Current naming convention. Defaults to hyphenated. + /// + public INamingConvention NamingConvention + { + get => _namingConvention; set + { + lock (_changeLock) + { + _namingConvention = value; + _changed = true; + } + + } + } + private List _typeConverters = new List { + new ListOrStringConverter(), + new YamlEnumConverter(), + }; + + private Dictionary _tagMappings = new Dictionary + { + { "!keyvault", typeof(object) } + }; + + private List _nodeDeserializers = new List + { + new TemplatedValueDeserializer() + }; + private bool _ignoreUnmatchedProperties; + + /// + /// Whether to ignore unmatched properties in the structure. + /// + public bool IgnoreUnmatchedProperties + { + get => _ignoreUnmatchedProperties; + set + { + lock (_changeLock) + { + if (_ignoreUnmatchedProperties != value) + { + _changed = true; + _ignoreUnmatchedProperties = value; + } + } + } + } + + private Dictionary _discriminatedUnions = new Dictionary(); + + /// + /// Maps the given tag to the type T. + /// Mapping is only required for custom tags. + /// + /// Tag to be mapped + /// Type to map to + public YamlConfigBuilder AddTagMapping(string tag) + { + if (tag == null) throw new ArgumentNullException(nameof(tag)); + lock (_changeLock) + { + _tagMappings[tag] = typeof(T); + _changed = true; + } + return this; + } + + /// + /// Adds a YAML type converter to the config deserializer. + /// + /// Type converter to add + public YamlConfigBuilder AddTypeConverter(IYamlTypeConverter converter) + { + if (converter == null) throw new ArgumentNullException(nameof(converter)); + lock (_changeLock) + { + _changed = true; + for (int i = 0; i < _typeConverters.Count; i++) + { + if (_typeConverters[i].GetType() == converter.GetType()) + { + _typeConverters[i] = converter; + return this; + } + } + _typeConverters.Add(converter); + } + return this; + } + + /// + /// Add a custom node deserializer to the config. + /// + /// Node deserializer to add + public YamlConfigBuilder AddNodeDeserializer(INodeDeserializer nodeDeserializer) + { + if (nodeDeserializer == null) throw new ArgumentNullException(nameof(nodeDeserializer)); + lock (_changeLock) + { + _changed = true; + for (int i = 0; i < _nodeDeserializers.Count; i++) + { + if (_nodeDeserializers[i].GetType() == nodeDeserializer.GetType()) + { + _nodeDeserializers[i] = nodeDeserializer; + return this; + } + } + _nodeDeserializers.Add(nodeDeserializer); + } + return this; + } + + /// + /// Add key vault support to the config loader, given a key vault config. + /// + /// + public YamlConfigBuilder AddKeyVault(KeyVaultConfig config) + { + if (config == null) throw new ArgumentNullException(nameof(config)); + + AddNodeDeserializer(new KeyVaultResolver(config)); + return this; + } + + + /// + /// Add an internally tagged type to the yaml deserializer. + /// + /// The type in your actual config structure that + /// indicates that this is a custom mapping. + /// + /// The key for the discriminator, i.e. "type" + /// A map from discriminator key value to type + public YamlConfigBuilder AddDiscriminatedType(string key, IDictionary variants) + { + lock (_changeLock) + { + _discriminatedUnions[typeof(TBase)] = new DiscriminatedUnionConfig(typeof(TBase), key, variants); + _changed = true; + } + return this; + } + + private IDeserializer Build() + { + var builder = new DeserializerBuilder() + .WithNamingConvention(NamingConvention); + + foreach (var tagMapping in _tagMappings) + { + builder = builder.WithTagMapping(tagMapping.Key, tagMapping.Value); + } + foreach (var deserializer in _nodeDeserializers) + { + builder = builder.WithNodeDeserializer(deserializer); + } + foreach (var converter in _typeConverters) + { + builder = builder.WithTypeConverter(converter); + } + if (IgnoreUnmatchedProperties) + { + builder = builder.IgnoreUnmatchedProperties(); + } + + if (_discriminatedUnions.Count > 0) + { + builder = builder.WithTypeDiscriminatingNodeDeserializer(o => + { + var method = o.GetType().GetMethod("AddKeyValueTypeDiscriminator"); + foreach (var union in _discriminatedUnions.Values) + { + method!.MakeGenericMethod(union.BaseType) + .Invoke(o, new object[] { union.Key, union.Variants }); + } + }); + } + + return builder.Build(); + } + + /// + /// Get a filtered serializer, used for displaying configuration objects + /// without logging secrets like passwords. + /// + /// Note, avoid using this on objects with cycles. + /// + /// List of items to keep even if they match defaults. + /// List of field names to ignore. You should put secrets and passwords in here + /// Prefixes on full type names for types that should be explored internally + /// Allow read only properties + /// + public ISerializer GetSafeSerializer( + IEnumerable toAlwaysKeep, + IEnumerable toIgnore, + IEnumerable namePrefixes, + bool allowReadOnly) + { + lock (_changeLock) + { + var builder = new SerializerBuilder() + .WithTypeInspector(insp => new DefaultFilterTypeInspector( + insp, + toAlwaysKeep, + toIgnore, + namePrefixes, + _typeConverters, + allowReadOnly)) + .WithNamingConvention(NamingConvention); + + foreach (var converter in _typeConverters) + { + builder.WithTypeConverter(converter); + } + + return builder.Build(); + } + } + } + + +} \ No newline at end of file diff --git a/ExtractorUtils.Test/unit/ConfigurationTest.cs b/ExtractorUtils.Test/unit/ConfigurationTest.cs index 27c20e86..dfda5a6c 100644 --- a/ExtractorUtils.Test/unit/ConfigurationTest.cs +++ b/ExtractorUtils.Test/unit/ConfigurationTest.cs @@ -427,5 +427,46 @@ public static void TestEnumConversion() Assert.Equal(TestEnum.Foo, ConfigurationUtils.ReadString("Foo")); Assert.Equal(TestEnum.Bar, ConfigurationUtils.ReadString("bar")); } + + class UnionWrapper + { + public List Items { get; set; } + } + + abstract class DiscriminatedUnionBase + { + public string Type { get; set; } + } + + class DiscriminatedUnionA : DiscriminatedUnionBase + { + public string FieldA { get; set; } + } + + class DiscriminatedUnionB : DiscriminatedUnionBase + { + public string FieldB { get; set; } + } + + [Fact] + public static void TestDiscriminatedUnion() + { + ConfigurationUtils.AddDiscriminatedType("type", new Dictionary { + { "typeA", typeof(DiscriminatedUnionA) }, + { "typeB", typeof(DiscriminatedUnionB) } + }); + + var val = ConfigurationUtils.ReadString(@"items: + - type: typeA + field-a: it's an A! + - type: typeB + field-b: it's a B! + "); + Assert.Equal(2, val.Items.Count); + var it = Assert.IsType(val.Items[0]); + Assert.Equal("it's an A!", it.FieldA); + var it2 = Assert.IsType(val.Items[1]); + Assert.Equal("it's a B!", it2.FieldB); + } } }