From 90a4574063339e712c69fbab2c8e96d6499240fc Mon Sep 17 00:00:00 2001 From: "H. C. Kruse" Date: Sun, 19 May 2024 20:18:09 +0200 Subject: [PATCH] feat: Parallelize item parsing --- Loader/EntityService.cs | 61 +++--- Loader/InventoryContainerLoader.cs | 3 +- Loader/InventoryContainerParser.cs | 44 ----- Loader/ItemLoader.cs | 112 ++++++----- Loader/MissionLoader.cs | 35 ++-- Loader/Program.cs | 37 ++-- Loader/ShipLoader.cs | 185 +++++++++++------- .../scdb.Xml/Entities/InventoryContainer.cs | 5 +- 8 files changed, 251 insertions(+), 231 deletions(-) delete mode 100644 Loader/InventoryContainerParser.cs diff --git a/Loader/EntityService.cs b/Loader/EntityService.cs index 724624b01..04a2a686d 100644 --- a/Loader/EntityService.cs +++ b/Loader/EntityService.cs @@ -1,8 +1,9 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; - +using System.Threading.Tasks; using Newtonsoft.Json; using scdb.Xml.Entities; @@ -14,11 +15,11 @@ public class EntityService public string OutputFolder { get; set; } public string DataRoot { get; set; } - Dictionary classNameToFilenameMap; - Dictionary referenceToClassNameMap; - Dictionary classNameToTypeMap; - Dictionary classNameToEntityMap; - ClassParser entityParser = new ClassParser(); + ConcurrentDictionary classNameToFilenameMap; + ConcurrentDictionary referenceToClassNameMap; + public ConcurrentDictionary classNameToTypeMap; + ConcurrentDictionary classNameToEntityMap; + ClassParser entityParser = new (); bool verbose = true; // Avoid filenames that have these endings @@ -51,20 +52,20 @@ public void Initialise(bool rebuildCache) var referenceCache = Path.Combine(DataRoot, "classReferences-scunpacked.json"); var typeCache = Path.Combine(DataRoot, "classTypes-scunpacked.json"); - classNameToEntityMap = new Dictionary(); + classNameToEntityMap = new ConcurrentDictionary(); if (!rebuildCache && File.Exists(filenameCache) && File.Exists(referenceCache) && File.Exists(typeCache)) { Console.WriteLine($"EntityService: Using the existing entity cache found in {DataRoot}"); var filenameContents = File.ReadAllText(filenameCache); - classNameToFilenameMap = JsonConvert.DeserializeObject>(filenameContents); + classNameToFilenameMap = JsonConvert.DeserializeObject>(filenameContents); var referenceContents = File.ReadAllText(referenceCache); - referenceToClassNameMap = JsonConvert.DeserializeObject>(referenceContents); + referenceToClassNameMap = JsonConvert.DeserializeObject>(referenceContents); var typeContents = File.ReadAllText(typeCache); - classNameToTypeMap = JsonConvert.DeserializeObject>(typeContents); + classNameToTypeMap = JsonConvert.DeserializeObject>(typeContents); } else { @@ -72,9 +73,9 @@ public void Initialise(bool rebuildCache) var timer = new System.Diagnostics.Stopwatch(); timer.Start(); - classNameToFilenameMap = new Dictionary(); - referenceToClassNameMap = new Dictionary(); - classNameToTypeMap = new Dictionary(); + classNameToFilenameMap = new ConcurrentDictionary(); + referenceToClassNameMap = new ConcurrentDictionary(); + classNameToTypeMap = new ConcurrentDictionary(); BuildItemDirectory(Path.Join("Data", "Libs", "Foundry", "Records", "entities", "scitem")); //BuildItemDirectory(@"Data\Libs\Foundry\Records\entities\scitem\ships"); @@ -99,7 +100,7 @@ public EntityClassDefinition GetByClassName(string className) if (classNameToFilenameMap.ContainsKey(className)) { var entity = LoadEntity(classNameToFilenameMap[className]); - classNameToEntityMap.Add(className, entity); + classNameToEntityMap.TryAdd(className, entity); return entity; } @@ -123,10 +124,10 @@ public EntityClassDefinition GetByFilename(string filename) if (entity == null) return null; if (!classNameToFilenameMap.ContainsKey(entity.ClassName)) { - classNameToFilenameMap.Add(entity.ClassName, filename); - referenceToClassNameMap.Add(entity.__ref, entity.ClassName); - classNameToEntityMap.Add(entity.ClassName, entity); - classNameToTypeMap.Add(entity.ClassName, entity.Components.SAttachableComponentParams?.AttachDef.Type ?? ""); + classNameToFilenameMap.TryAdd(entity.ClassName, filename); + referenceToClassNameMap.TryAdd(entity.__ref, entity.ClassName); + classNameToEntityMap.TryAdd(entity.ClassName, entity); + classNameToTypeMap.TryAdd(entity.ClassName, entity.Components.SAttachableComponentParams?.AttachDef.Type ?? ""); } return entity; @@ -146,17 +147,19 @@ public IEnumerable GetAll(string typeFilter) void BuildItemDirectory(string folder) { - foreach (var filename in Directory.EnumerateFiles(Path.Combine(DataRoot, folder), "*.xml", SearchOption.AllDirectories)) - { - if (avoidFile(filename)) continue; - - var entity = LoadEntity(filename); - - classNameToFilenameMap.Add(entity.ClassName, filename); - referenceToClassNameMap.Add(entity.__ref, entity.ClassName); - classNameToEntityMap.Add(entity.ClassName, entity); - classNameToTypeMap.Add(entity.ClassName, entity.Components?.SAttachableComponentParams?.AttachDef.Type ?? ""); - } + Parallel.ForEach( + Directory.EnumerateFiles(Path.Combine(DataRoot, folder), "*.xml", SearchOption.AllDirectories), + filename => + { + if (avoidFile(filename)) return; + + var entity = LoadEntity(filename); + + classNameToFilenameMap.TryAdd(entity.ClassName, filename); + referenceToClassNameMap.TryAdd(entity.__ref, entity.ClassName); + classNameToEntityMap.TryAdd(entity.ClassName, entity); + classNameToTypeMap.TryAdd(entity.ClassName, entity.Components?.SAttachableComponentParams?.AttachDef.Type ?? ""); + }); } EntityClassDefinition LoadEntity(string filename) diff --git a/Loader/InventoryContainerLoader.cs b/Loader/InventoryContainerLoader.cs index 56e0b99f1..c9ec95178 100644 --- a/Loader/InventoryContainerLoader.cs +++ b/Loader/InventoryContainerLoader.cs @@ -3,6 +3,7 @@ using System.IO; using Newtonsoft.Json; +using scdb.Xml.Entities; namespace Loader { @@ -23,7 +24,7 @@ public List Load() List Load(string entityFolder) { var index = new List(); - var parser = new InventoryContainerParser(); + var parser = new ClassParser(); foreach (var entityFilename in Directory.EnumerateFiles(Path.Combine(DataRoot, entityFolder), "*.xml", SearchOption.AllDirectories)) { diff --git a/Loader/InventoryContainerParser.cs b/Loader/InventoryContainerParser.cs deleted file mode 100644 index 6e36b4f2d..000000000 --- a/Loader/InventoryContainerParser.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.Xml; -using System.Xml.Serialization; -using System.IO; - -using scdb.Xml.Entities; - -namespace Loader -{ - public class InventoryContainerParser - { - public InventoryContainer Parse(string fullXmlPath) - { - if (!File.Exists(fullXmlPath)) - { - Console.WriteLine("InventoryContainer file does not exist"); - return null; - } - - return ParseInventoryContainer(fullXmlPath); - } - - InventoryContainer ParseInventoryContainer(string xmlFilename) - { - string rootNodeName; - using (var reader = XmlReader.Create(new StreamReader(xmlFilename))) - { - reader.MoveToContent(); - rootNodeName = reader.Name; - } - - var xml = File.ReadAllText(xmlFilename); - var doc = new XmlDocument(); - doc.LoadXml(xml); - - var serialiser = new XmlSerializer(typeof(InventoryContainer), new XmlRootAttribute { ElementName = rootNodeName }); - using (var stream = new XmlNodeReader(doc)) - { - var entity = (InventoryContainer)serialiser.Deserialize(stream); - return entity; - } - } - } -} diff --git a/Loader/ItemLoader.cs b/Loader/ItemLoader.cs index 9b2fc0a34..0d6d503bb 100644 --- a/Loader/ItemLoader.cs +++ b/Loader/ItemLoader.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.IO; using System.Collections.Generic; using System.Linq; - +using System.Threading; +using System.Threading.Tasks; using Newtonsoft.Json; using scdb.Xml.Entities; @@ -14,19 +16,20 @@ public class ItemLoader public string OutputFolder { get; set; } public string DataRoot { get; set; } - ItemBuilder itemBuilder; - ManufacturerService manufacturerSvc; - ItemClassifier itemClassifier; - EntityService entitySvc; - AmmoService ammoSvc; - ItemInstaller itemInstaller; - LoadoutLoader loadoutLoader; - InventoryContainerService _inventoryContainerSvc; - MeleeCombatService meleeConfigSvc; + private readonly ItemBuilder _itemBuilder; + private readonly ManufacturerService _manufacturerSvc; + private readonly ItemClassifier _itemClassifier; + private readonly EntityService _entitySvc; + private readonly AmmoService _ammoSvc; + private readonly ItemInstaller _itemInstaller; + private readonly LoadoutLoader _loadoutLoader; + private readonly InventoryContainerService _inventoryContainerSvc; + private readonly MeleeCombatService _meleeConfigSvc; + private readonly bool _doV2Style; // Don't dump items with these types - string[] type_avoids = - { + private readonly string[] _typeAvoids = + [ "UNDEFINED", "airtrafficcontroller", "button", @@ -45,25 +48,29 @@ public class ItemLoader "flair_surface", "flair_wall", "removablechip", - }; + ]; - public ItemLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, EntityService entitySvc, AmmoService ammoSvc, ItemInstaller itemInstaller, LoadoutLoader loadoutLoader, InventoryContainerService inventoryContainerSvc, MeleeCombatService meleeConfigSvc) + public ItemLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, EntityService entitySvc, AmmoService ammoSvc, ItemInstaller itemInstaller, LoadoutLoader loadoutLoader, InventoryContainerService inventoryContainerSvc, MeleeCombatService meleeConfigSvc, bool doV2Style) { - this.itemBuilder = itemBuilder; - this.manufacturerSvc = manufacturerSvc; - itemClassifier = new ItemClassifier(); - this.entitySvc = entitySvc; - this.ammoSvc = ammoSvc; - this.itemInstaller = itemInstaller; - this.loadoutLoader = loadoutLoader; - this.meleeConfigSvc = meleeConfigSvc; + _itemBuilder = itemBuilder; + _manufacturerSvc = manufacturerSvc; + _itemClassifier = new ItemClassifier(); + _entitySvc = entitySvc; + _ammoSvc = ammoSvc; + _itemInstaller = itemInstaller; + _loadoutLoader = loadoutLoader; + _meleeConfigSvc = meleeConfigSvc; _inventoryContainerSvc = inventoryContainerSvc; + _doV2Style = doV2Style; } - public List Load(string typeFilter = null) + public ConcurrentBag Load(string typeFilter = null) { Directory.CreateDirectory(Path.Combine(OutputFolder, "items")); - Directory.CreateDirectory(Path.Combine(OutputFolder, "v2", "items")); + if (_doV2Style) + { + Directory.CreateDirectory(Path.Combine(OutputFolder, "v2", "items")); + } var damageResistanceMacros = LoadDamageResistanceMacros(); @@ -75,13 +82,13 @@ public List Load(string typeFilter = null) Console.WriteLine($"ItemLoader: Creating {index.Count} item files..."); foreach (var item in index) { - var entity = entitySvc.GetByClassName(item.className); + var entity = _entitySvc.GetByClassName(item.className); // If uses an ammunition magazine, then load it EntityClassDefinition magazine = null; if (!String.IsNullOrEmpty(entity.Components?.SCItemWeaponComponentParams?.ammoContainerRecord)) { - magazine = entitySvc.GetByReference(entity.Components.SCItemWeaponComponentParams.ammoContainerRecord); + magazine = _entitySvc.GetByReference(entity.Components.SCItemWeaponComponentParams.ammoContainerRecord); } // If it is an ammo container or if it has a magazine then load the ammo properties @@ -89,14 +96,14 @@ public List Load(string typeFilter = null) var ammoRef = magazine?.Components?.SAmmoContainerComponentParams?.ammoParamsRecord ?? entity.Components?.SAmmoContainerComponentParams?.ammoParamsRecord; if (!String.IsNullOrEmpty(ammoRef)) { - ammoEntry = ammoSvc.GetByReference(ammoRef); + ammoEntry = _ammoSvc.GetByReference(ammoRef); } MeleeCombatConfig combatConfig = null; var combatRef = entity?.Components?.SMeleeWeaponComponentParams?.meleeCombatConfig; if (!String.IsNullOrEmpty(combatRef)) { - combatConfig = meleeConfigSvc.GetByReference(combatRef); + combatConfig = _meleeConfigSvc.GetByReference(combatRef); } DamageResistance damageResistances = null; @@ -112,23 +119,24 @@ public List Load(string typeFilter = null) inventoryContainer = _inventoryContainerSvc.GetInventoryContainer(entity.Components.SCItemInventoryContainerComponentParams.containerParams); } - var stdItem = itemBuilder.BuildItem(entity); - var loadout = loadoutLoader.Load(entity); - itemInstaller.InstallLoadout(stdItem, loadout); - itemInstaller.InstallLoadout(entity, loadout); + var stdItem = _itemBuilder.BuildItem(entity); + var loadout = _loadoutLoader.Load(entity); + _itemInstaller.InstallLoadout(stdItem, loadout); + _itemInstaller.InstallLoadout(entity, loadout); stdItem.Classification = item.classification; item.stdItem = stdItem; - File.WriteAllText(Path.Combine(OutputFolder, "v2", "items", $"{entity.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(stdItem)); - File.WriteAllText(Path.Combine(OutputFolder, "v2", "items", $"{entity.ClassName.ToLower()}-raw.json"), JsonConvert.SerializeObject(entity)); + if (_doV2Style) + { + File.WriteAllText(Path.Combine(OutputFolder, "v2", "items", $"{entity.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(stdItem)); + File.WriteAllText(Path.Combine(OutputFolder, "v2", "items", $"{entity.ClassName.ToLower()}-raw.json"), JsonConvert.SerializeObject(entity)); + } // Write the JSON of this entity to its own file var jsonFilename = Path.Combine(OutputFolder, "items", $"{entity.ClassName.ToLower()}.json"); var json = JsonConvert.SerializeObject(new { - magazine, - ammo = ammoEntry, Raw = new { Entity = entity, @@ -136,6 +144,8 @@ public List Load(string typeFilter = null) damageResistances, inventoryContainer, combatConfig, + magazine, + ammo = ammoEntry, }); File.WriteAllText(jsonFilename, json); } @@ -178,27 +188,35 @@ private List LoadDamageResistanceMacros() return damageResistanceMacros; } - List CreateIndex(string typeFilter) + ConcurrentBag CreateIndex(string typeFilter) { - var index = new List(); + var timer = new System.Diagnostics.Stopwatch(); + timer.Start(); + var index = new ConcurrentBag (); + var types = typeFilter?.Split(',') ?? new string[0]; - foreach (var entity in entitySvc.GetAll(typeFilter)) + Parallel.ForEach(_entitySvc.classNameToTypeMap, entity => { - // Skip types that are not very interesting - if (avoidType(entity.Components?.SAttachableComponentParams?.AttachDef.Type)) continue; + if (types.Length > 0 && !types.Contains(entity.Value)) return; - var indexEntry = CreateIndexEntry(entity); + var entityDef = _entitySvc.GetByClassName(entity.Key); + if (AvoidType(entityDef.Components?.SAttachableComponentParams?.AttachDef.Type)) return; + var indexEntry = CreateIndexEntry(entityDef); // Add it to the item index index.Add(indexEntry); - } + }); + + timer.Stop(); + + Console.WriteLine($"ItemLoader: Loading items took {timer.Elapsed.TotalMinutes:n1} minutes"); return index; } ItemIndexEntry CreateIndexEntry(EntityClassDefinition entity) { - var classification = itemClassifier.Classify(entity); + var classification = _itemClassifier.Classify(entity); var indexEntry = new ItemIndexEntry { @@ -211,16 +229,16 @@ ItemIndexEntry CreateIndexEntry(EntityClassDefinition entity) grade = entity.Components?.SAttachableComponentParams?.AttachDef.Grade, name = entity.Components?.SAttachableComponentParams?.AttachDef.Localization.Name, tags = entity.Components?.SAttachableComponentParams?.AttachDef.Tags, - manufacturer = manufacturerSvc.GetManufacturer(entity.Components?.SAttachableComponentParams?.AttachDef.Manufacturer, entity.ClassName)?.Code, + manufacturer = _manufacturerSvc.GetManufacturer(entity.Components?.SAttachableComponentParams?.AttachDef.Manufacturer, entity.ClassName)?.Code, classification = classification }; return indexEntry; } - bool avoidType(string type) + private bool AvoidType(string type) { if (type == null) return true; - return type_avoids.Contains(type, StringComparer.OrdinalIgnoreCase); + return _typeAvoids.Contains(type, StringComparer.OrdinalIgnoreCase); } } } diff --git a/Loader/MissionLoader.cs b/Loader/MissionLoader.cs index d6c9c8e48..c31799c0e 100644 --- a/Loader/MissionLoader.cs +++ b/Loader/MissionLoader.cs @@ -1,5 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Concurrent; using System.IO; +using System.Threading.Tasks; using Loader.scdb.Xml.Missiongiver; using Loader.scdb.Xml.Missiontype; using Newtonsoft.Json; @@ -13,64 +14,64 @@ public class MissionLoader public string DataRoot { get; set; } public LocalisationService locService { get; set; } - public Dictionary LoadMissions() + public ConcurrentDictionary LoadMissions() { Directory.CreateDirectory(Path.Combine(OutputFolder, "missions")); - var output = new Dictionary(); + var output = new ConcurrentDictionary(); var parser = new ClassParser(); var path = Path.Combine(DataRoot, Path.Join("Data", "Libs", "Foundry", "Records", "missionbroker")); - foreach (var entityFilename in Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories)) + Parallel.ForEach(Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories), entityFilename => { var mission = parser.Parse(entityFilename); AddTranslations(mission); - output.Add(mission.__ref, mission); + output.TryAdd(mission.__ref, mission); File.WriteAllText(Path.Combine(OutputFolder, "missions", $"{mission.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(mission)); - } + }); return output; } - public Dictionary LoadMissionTypes() + public ConcurrentDictionary LoadMissionTypes() { Directory.CreateDirectory(Path.Combine(OutputFolder, "missions", "types")); - var output = new Dictionary(); + var output = new ConcurrentDictionary(); var path = Path.Combine(DataRoot, Path.Join("Data", "Libs", "Foundry", "Records", "missiontype")); var parser = new ClassParser(); - foreach (var entityFilename in Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories)) + Parallel.ForEach(Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories), entityFilename => { var missionType = parser.Parse(entityFilename); AddTypeTranslations(missionType); - output.Add(missionType.__ref, missionType); - File.WriteAllText(Path.Combine(OutputFolder, "missions", "types", $"{missionType.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(missionType)); - } + output.TryAdd(missionType.__ref, missionType); + File.WriteAllText(Path.Combine(OutputFolder, "missions", "types", $"{missionType.ClassName.ToLower()}.json"),JsonConvert.SerializeObject(missionType)); + }); return output; } - public Dictionary LoadMissionGiver() + public ConcurrentDictionary LoadMissionGiver() { Directory.CreateDirectory(Path.Combine(OutputFolder, "missions", "missiongiver")); - var output = new Dictionary(); + var output = new ConcurrentDictionary(); var path = Path.Combine(DataRoot, Path.Join("Data", "Libs", "Foundry", "Records", "missiongiver")); var parser = new ClassParser(); - foreach (var entityFilename in Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories)) + Parallel.ForEach(Directory.EnumerateFiles(path, "*.xml", SearchOption.AllDirectories), entityFilename => { var giver = parser.Parse(entityFilename); AddGiverTranslations(giver); - output.Add(giver.__ref, giver); + output.TryAdd(giver.__ref, giver); File.WriteAllText(Path.Combine(OutputFolder, "missions", "missiongiver", $"{giver.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(giver)); - } + }); return output; } diff --git a/Loader/Program.cs b/Loader/Program.cs index 1589d241c..4b07877a6 100644 --- a/Loader/Program.cs +++ b/Loader/Program.cs @@ -15,12 +15,13 @@ static void Main(string[] args) { string scDataRoot = null; string outputRoot = null; - bool doShips = true; - bool doItems = true; - bool doShops = true; - bool doStarmap = true; - bool doMissions = true; - bool noCache = false; + var doShips = true; + var doItems = true; + var doShops = true; + var doStarmap = true; + var doMissions = true; + var noCache = false; + var doV2Style = true; string typeFilter = null; string shipFilter = null; @@ -29,12 +30,13 @@ static void Main(string[] args) { "scdata=", v => scDataRoot = v }, { "input=", v => scDataRoot = v }, { "output=", v => outputRoot = v }, - { "noships", v => doShips = false }, - { "noitems", v => doItems = false }, - { "noshops", v => doShops = false }, - { "nomap", v => doStarmap = false }, - { "nomissions", v => doMissions = false }, - { "nocache", v => noCache = true }, + { "noships", _ => doShips = false }, + { "noitems", _ => doItems = false }, + { "noshops", _ => doShops = false }, + { "nomap", _ => doStarmap = false }, + { "noV2", _ => doV2Style = false }, + { "nomissions", _ => doMissions = false }, + { "nocache", _ => noCache = true }, { "types=", v => typeFilter = v }, { "ships=", v=> shipFilter = v } }; @@ -50,6 +52,9 @@ static void Main(string[] args) return; } + var timer = new System.Diagnostics.Stopwatch(); + timer.Start(); + JsonConvert.DefaultSettings = () => new JsonSerializerSettings { Formatting = Formatting.Indented, @@ -186,7 +191,7 @@ static void Main(string[] args) if (doItems) { Console.WriteLine("Load Items"); - var itemLoader = new ItemLoader(itemBuilder, manufacturerSvc, entitySvc, ammoSvc, itemInstaller, loadoutLoader, inventorySvc, meleeConfigSvc) + var itemLoader = new ItemLoader(itemBuilder, manufacturerSvc, entitySvc, ammoSvc, itemInstaller, loadoutLoader, inventorySvc, meleeConfigSvc, doV2Style) { OutputFolder = outputRoot, DataRoot = scDataRoot, @@ -198,7 +203,7 @@ static void Main(string[] args) if (doShips) { Console.WriteLine("Load Ships and Vehicles"); - var shipLoader = new ShipLoader(itemBuilder, manufacturerSvc, localisationSvc, entitySvc, itemInstaller, loadoutLoader, null, inventorySvc)//insuranceSvc) + var shipLoader = new ShipLoader(itemBuilder, manufacturerSvc, localisationSvc, entitySvc, itemInstaller, loadoutLoader, inventorySvc, doV2Style) { OutputFolder = outputRoot, DataRoot = scDataRoot, @@ -230,7 +235,9 @@ static void Main(string[] args) starmapLoader.Load(); } - Console.WriteLine("Finished!"); + timer.Stop(); + + Console.WriteLine($"Finished! Took {timer.Elapsed.TotalMinutes:n1} minutes"); } } } diff --git a/Loader/ShipLoader.cs b/Loader/ShipLoader.cs index 919020111..1f80df0e8 100644 --- a/Loader/ShipLoader.cs +++ b/Loader/ShipLoader.cs @@ -1,13 +1,11 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; -using System.Collections.Generic; using System.Text; using System.Text.RegularExpressions; -using System.Globalization; - using Newtonsoft.Json; - using scdb.Xml.Entities; using scdb.Xml.Vehicles; @@ -18,8 +16,8 @@ public class ShipLoader public string OutputFolder { get; set; } public string DataRoot { get; set; } - string[] avoids = - { + private readonly string[] _avoids = + [ // CIG tags "pu", "ai", @@ -47,36 +45,49 @@ public class ShipLoader "advocacy", "derelict", "drone", - "eaobjectivedestructable", - }; - - SortedSet AcceptTypes = new SortedSet(); - SortedSet InstalledTypes = new SortedSet(); - ItemBuilder itemBuilder; - ManufacturerService manufacturerSvc; - LocalisationService localisationSvc; - EntityService entitySvc; - ItemInstaller itemInstaller; - LoadoutLoader loadoutLoader; - InsuranceService insuranceSvc; - InventoryContainerService inventoryContainerSvc; - - public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, LocalisationService localisationSvc, EntityService entitySvc, ItemInstaller itemInstaller, LoadoutLoader loadoutLoader, InsuranceService insuranceSvc, InventoryContainerService inventoryContainerSvc) - { - this.itemBuilder = itemBuilder; - this.manufacturerSvc = manufacturerSvc; - this.localisationSvc = localisationSvc; - this.entitySvc = entitySvc; - this.itemInstaller = itemInstaller; - this.loadoutLoader = loadoutLoader; - this.insuranceSvc = insuranceSvc; - this.inventoryContainerSvc = inventoryContainerSvc; + "eaobjectivedestructable" + ]; + + private readonly SortedSet _acceptTypes = []; + private readonly SortedSet _installedTypes = []; + private readonly ItemBuilder _itemBuilder; + private readonly ManufacturerService _manufacturerSvc; + private readonly LocalisationService _localisationSvc; + private readonly EntityService _entitySvc; + private readonly ItemInstaller _itemInstaller; + private readonly LoadoutLoader _loadoutLoader; + private readonly InventoryContainerService _inventoryContainerSvc; + private readonly bool _doV2Style; + private Dictionary _vehicleImplementations; + + public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, LocalisationService localisationSvc, EntityService entitySvc, ItemInstaller itemInstaller, LoadoutLoader loadoutLoader, InventoryContainerService inventoryContainerSvc, bool doV2Style) + { + _itemBuilder = itemBuilder; + _manufacturerSvc = manufacturerSvc; + _localisationSvc = localisationSvc; + _entitySvc = entitySvc; + _itemInstaller = itemInstaller; + _loadoutLoader = loadoutLoader; + _inventoryContainerSvc = inventoryContainerSvc; + _doV2Style = doV2Style; } public List<(ShipIndexEntry, StandardisedShip)> Load(string shipFilter) { + var implPath = Path.Combine(DataRoot, "Data", "Scripts", "Entities", "Vehicles", "Implementations", "Xml"); + var mappings = new Dictionary(); + foreach (var file in Directory.EnumerateFiles(implPath, "*.xml")) + { + mappings.Add(file.Split("/").Last().ToLower(), file); + } + + _vehicleImplementations = mappings; + Directory.CreateDirectory(Path.Combine(OutputFolder, "ships")); - Directory.CreateDirectory(Path.Combine(OutputFolder, "v2", "ships")); + if (_doV2Style) + { + Directory.CreateDirectory(Path.Combine(OutputFolder, "v2", "ships")); + } var index = new List<(ShipIndexEntry, StandardisedShip)>(); index.AddRange(LoadFolder(Path.Join("Data", "Libs", "Foundry", "Records", "entities", "spaceships"), shipFilter)); @@ -85,19 +96,26 @@ public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, var oldIndexItems = index.Select(x => x.Item1).ToList(); var newIndexItems = index.Select(x => x.Item2).ToList(); - File.WriteAllText(Path.Combine(OutputFolder, "ships.json"), JsonConvert.SerializeObject(oldIndexItems)); - File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships.json"), JsonConvert.SerializeObject(newIndexItems)); + if (_doV2Style) + { + File.WriteAllText(Path.Combine(OutputFolder, "ships.json"), JsonConvert.SerializeObject(oldIndexItems)); + File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships.json"), JsonConvert.SerializeObject(newIndexItems)); + } + else + { + File.WriteAllText(Path.Combine(OutputFolder, "ships.json"), JsonConvert.SerializeObject(newIndexItems)); + } Console.WriteLine(); Console.WriteLine("*** All accepted types ***"); - foreach (var type in AcceptTypes) + foreach (var type in _acceptTypes) { Console.WriteLine(type); } Console.WriteLine(); Console.WriteLine("*** All installed types ***"); - foreach (var type in InstalledTypes) + foreach (var type in _installedTypes) { Console.WriteLine(type); } @@ -123,18 +141,7 @@ public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, (var vehicle, var entity, var parts, var ship, var ports) = shipTuple.Value; - File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(ship)); - File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-parts.json"), JsonConvert.SerializeObject(parts)); - File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-ports.json"), JsonConvert.SerializeObject(ports)); - - var v2json = JsonConvert.SerializeObject(new - { - Entity = entity, - Vehicle = vehicle - }); - File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-raw.json"), v2json); - - var v1json = JsonConvert.SerializeObject(new + var v1Json = JsonConvert.SerializeObject(new { Raw = new { @@ -142,7 +149,28 @@ public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, Vehicle = vehicle, } }); - File.WriteAllText(Path.Combine(OutputFolder, "ships", $"{entity.ClassName.ToLower()}.json"), v1json); + + if (_doV2Style) + { + var v2Json = JsonConvert.SerializeObject(new + { + Entity = entity, + Vehicle = vehicle + }); + + File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(ship)); + File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-parts.json"), JsonConvert.SerializeObject(parts)); + File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-ports.json"), JsonConvert.SerializeObject(ports)); + File.WriteAllText(Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-raw.json"), v2Json); + + File.WriteAllText(Path.Combine(OutputFolder, "ships", $"{entity.ClassName.ToLower()}.json"), v1Json); + } + else + { + File.WriteAllText(Path.Combine(OutputFolder, "ships", $"{entity.ClassName.ToLower()}-raw.json"), v1Json); + File.WriteAllText(Path.Combine(OutputFolder, "ships", $"{entity.ClassName.ToLower()}.json"), JsonConvert.SerializeObject(ship)); + } + // Index entry var indexEntry = CreateIndexEntry(entity, vehicle, ship); @@ -156,8 +184,6 @@ public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, (Vehicle, EntityClassDefinition, List, StandardisedShip, StandardisedPortSummary)? LoadShip(string entityFilename) { - Console.WriteLine(entityFilename); - var entity = LoadEntity(entityFilename); var vehicle = LoadVehicle(entity); @@ -178,7 +204,7 @@ public ShipLoader(ItemBuilder itemBuilder, ManufacturerService manufacturerSvc, bool avoidFile(string filename) { var fileSplit = Path.GetFileNameWithoutExtension(filename).Split('_'); - return fileSplit.Any(part => avoids.Contains(part)); + return fileSplit.Any(part => _avoids.Contains(part)); } ShipIndexEntry CreateIndexEntry(EntityClassDefinition entity, Vehicle vehicle, StandardisedShip shipSummary) @@ -186,15 +212,15 @@ ShipIndexEntry CreateIndexEntry(EntityClassDefinition entity, Vehicle vehicle, S bool isGroundVehicle = entity.Components?.VehicleComponentParams.vehicleCareer == "@vehicle_focus_ground"; bool isGravlevVehicle = entity.Components?.VehicleComponentParams.isGravlevVehicle ?? false; bool isSpaceship = !(isGroundVehicle || isGravlevVehicle); - var manufacturer = manufacturerSvc.GetManufacturer(entity.Components?.VehicleComponentParams.manufacturer, entity.ClassName); + var manufacturer = _manufacturerSvc.GetManufacturer(entity.Components?.VehicleComponentParams?.manufacturer, entity.ClassName); var indexEntry = new ShipIndexEntry { className = entity.ClassName, - name = entity.Components?.VehicleComponentParams.vehicleName, - career = entity.Components?.VehicleComponentParams.vehicleCareer, - role = entity.Components?.VehicleComponentParams.vehicleRole, - dogFightEnabled = Convert.ToBoolean(entity.Components?.VehicleComponentParams.dogfightEnabled), + name = entity.Components?.VehicleComponentParams?.vehicleName, + career = entity.Components?.VehicleComponentParams?.vehicleCareer, + role = entity.Components?.VehicleComponentParams?.vehicleRole, + dogFightEnabled = Convert.ToBoolean(entity.Components?.VehicleComponentParams?.dogfightEnabled), size = shipSummary?.Size, isGroundVehicle = isGroundVehicle, isGravlevVehicle = isGravlevVehicle, @@ -209,7 +235,7 @@ ShipIndexEntry CreateIndexEntry(EntityClassDefinition entity, Vehicle vehicle, S EntityClassDefinition LoadEntity(string entityFilename) { - var entity = entitySvc.GetByFilename(entityFilename); + var entity = _entitySvc.GetByFilename(entityFilename); return entity; } @@ -220,7 +246,16 @@ Vehicle LoadVehicle(EntityClassDefinition entity) return null; } - vehicleFilename = Path.Combine(DataRoot, "Data", Path.Join(vehicleFilename.Split('/'))); + var search = vehicleFilename.Split('/').Last().ToLower(); + + if (!_vehicleImplementations.ContainsKey(search.ToLower())) + { + Console.WriteLine("Vehicle implementation " + search + " not Found."); + return null; + } + + vehicleFilename = _vehicleImplementations[search.ToLower()]; + var vehicleModification = entity.Components?.VehicleComponentParams?.modification; if (String.IsNullOrEmpty(vehicleModification)) Console.WriteLine(vehicleFilename); @@ -271,22 +306,25 @@ void CreatePartsDump(EntityClassDefinition entity, List parts) sb.Append($"{port.Category,-40}"); sb.AppendLine(); - if (port.Types != null) foreach (var t in port.Types) AcceptTypes.Add(t); - if (port.InstalledItem?.Type != null) InstalledTypes.Add(port.InstalledItem.Type); + if (port.Types != null) foreach (var t in port.Types) _acceptTypes.Add(t); + if (port.InstalledItem?.Type != null) _installedTypes.Add(port.InstalledItem.Type); } sb.AppendLine(); - var newFilename = Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-dump.txt"); - File.WriteAllText(newFilename, sb.ToString()); + if (_doV2Style) + { + var newFilename = Path.Combine(OutputFolder, "v2", "ships", $"{entity.ClassName.ToLower()}-dump.txt"); + File.WriteAllText(newFilename, sb.ToString()); + } } List InitialiseShip(EntityClassDefinition entity, Vehicle vehicle) { - var loadout = loadoutLoader.Load(entity); + var loadout = _loadoutLoader.Load(entity); var partList = vehicle != null ? BuildPartList(vehicle.Parts) : DeducePartList(loadout); - itemInstaller.InstallLoadout(partList, loadout); + _itemInstaller.InstallLoadout(partList, loadout); InstallFakeItems(partList); @@ -386,13 +424,13 @@ List BuildPortTypes(Part part) var major = partType.type; if (String.IsNullOrWhiteSpace(major)) continue; - if (String.IsNullOrWhiteSpace(partType.subtypes)) types.Add(itemBuilder.BuildTypeName(major, null)); + if (String.IsNullOrWhiteSpace(partType.subtypes)) types.Add(_itemBuilder.BuildTypeName(major, null)); else { foreach (var subType in partType.subtypes.Split(",")) { var minor = subType; - types.Add(itemBuilder.BuildTypeName(major, minor)); + types.Add(_itemBuilder.BuildTypeName(major, minor)); } } } @@ -512,20 +550,19 @@ StandardisedPortSummary BuildPortSummary(List parts) StandardisedShip BuildShipSummary(EntityClassDefinition entity, List parts, StandardisedPortSummary portSummary, Vehicle vehicle) { StandardisedInventoryContainer inventorySize = null; - if (entity.Components?.VehicleComponentParams.inventoryContainerParams.Length > 0) + if (entity.Components?.VehicleComponentParams?.inventoryContainerParams?.Length > 0) { - inventorySize = inventoryContainerSvc.GetInventoryContainer(entity.Components?.VehicleComponentParams - .inventoryContainerParams); + inventorySize = _inventoryContainerSvc.GetInventoryContainer(entity.Components?.VehicleComponentParams?.inventoryContainerParams); } var shipSummary = new StandardisedShip { ClassName = entity.ClassName, - Name = localisationSvc.GetText(entity.Components?.VehicleComponentParams.vehicleName, entity.ClassName), - Description = localisationSvc.GetText(entity.Components?.VehicleComponentParams.vehicleDescription), - Career = localisationSvc.GetText(entity.Components?.VehicleComponentParams.vehicleCareer), - Role = localisationSvc.GetText(entity.Components?.VehicleComponentParams.vehicleRole), - Manufacturer = manufacturerSvc.GetManufacturer(entity.Components?.VehicleComponentParams.manufacturer, entity.ClassName), + Name = _localisationSvc.GetText(entity.Components?.VehicleComponentParams?.vehicleName, entity.ClassName), + Description = _localisationSvc.GetText(entity.Components?.VehicleComponentParams?.vehicleDescription), + Career = _localisationSvc.GetText(entity.Components?.VehicleComponentParams?.vehicleCareer), + Role = _localisationSvc.GetText(entity.Components?.VehicleComponentParams?.vehicleRole), + Manufacturer = _manufacturerSvc.GetManufacturer(entity.Components?.VehicleComponentParams?.manufacturer, entity.ClassName), Size = entity.Components?.SAttachableComponentParams?.AttachDef?.Size ?? 0, Width = entity.Components?.VehicleComponentParams?.maxBoundingBoxSize?.x ?? 0, Length = entity.Components?.VehicleComponentParams?.maxBoundingBoxSize?.y ?? 0, diff --git a/Loader/scdb.Xml/Entities/InventoryContainer.cs b/Loader/scdb.Xml/Entities/InventoryContainer.cs index 856e37e15..e58d437c7 100644 --- a/Loader/scdb.Xml/Entities/InventoryContainer.cs +++ b/Loader/scdb.Xml/Entities/InventoryContainer.cs @@ -2,11 +2,8 @@ namespace scdb.Xml.Entities { - public class InventoryContainer + public class InventoryContainer: ClassBase { - [XmlAttribute] - public string __ref; - public InteriorDimensions interiorDimensions; public InventoryType inventoryType; }