From 89c6e2d323837b7b9453c7082c93d090b1c56331 Mon Sep 17 00:00:00 2001 From: ovska Date: Sun, 27 Oct 2024 15:03:15 +0200 Subject: [PATCH 01/12] feat: cancellation support to integrity checker --- Pack3r.Core/Services/IntegrityChecker.cs | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/Pack3r.Core/Services/IntegrityChecker.cs b/Pack3r.Core/Services/IntegrityChecker.cs index 585d30a..555e7f3 100644 --- a/Pack3r.Core/Services/IntegrityChecker.cs +++ b/Pack3r.Core/Services/IntegrityChecker.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using NAudio.Wave; using Pack3r.Extensions; +using Pack3r.IO; using Pack3r.Logging; using Pack3r.Models; @@ -13,18 +14,18 @@ public interface IIntegrityChecker void CheckIntegrity(IAsset asset); } -public sealed class IntegrityChecker(ILogger logger) : IIntegrityChecker +public sealed class IntegrityChecker(ILogger logger, AppLifetime lifetime) : IIntegrityChecker { public void Log() { if (!_jpgs.IsEmpty) { - logger.Warn($"Found potentially progressive JPGs which are unsupported on 2.60b: {Format(_jpgs)}"); + logger.Warn($"Found potentially progressive JPGs which are unsupported on 2.60b:{Format(_jpgs)}"); } if (!_tgas.IsEmpty) { - logger.Warn($"Found top-left pixel ordered TGAs which are drawn upside down on 2.60b clients: {Format(_tgas)}"); + logger.Warn($"Found top-left pixel ordered TGAs which are drawn upside down on 2.60b clients:{Format(_tgas)}"); } foreach (var (path, warning) in _wavs) @@ -82,7 +83,7 @@ private void CheckIntegrityCore(IAsset asset) if (extension.EqualsF(".jpg")) { - VerifyJpg(fullPath, stream); + VerifyJpg(fullPath, stream, useAsync: asset.Source is not Pk3AssetSource); return; } @@ -95,6 +96,8 @@ private void CheckIntegrityCore(IAsset asset) private void VerifyTga(string path, Stream stream) { + lifetime.CancellationToken.ThrowIfCancellationRequested(); + const int index = 17; int value = -1; @@ -121,12 +124,21 @@ private void VerifyTga(string path, Stream stream) } } - private void VerifyJpg(string fullPath, Stream stream) + private void VerifyJpg(string fullPath, Stream stream, bool useAsync) { + lifetime.CancellationToken.ThrowIfCancellationRequested(); + // some light testing determined the average jpg to be <85kb - using var ms = Global.StreamManager.GetStream(nameof(VerifyJpg), requiredSize: 1024 * 128); + using var ms = Global.StreamManager.GetStream(nameof(VerifyJpg), requiredSize: 1024 * 128, asContiguousBuffer: true); - stream.CopyTo(ms); + if (useAsync) + { + stream.CopyToAsync(ms, lifetime.CancellationToken).GetAwaiter().GetResult(); + } + else + { + stream.CopyTo(ms); + } if (!ms.TryGetBuffer(out ArraySegment buffer)) buffer = ms.ToArray(); @@ -141,7 +153,7 @@ private void VerifyJpg(string fullPath, Stream stream) if (bytePairs.IndexOf(DCTbytes) >= 0 && bytePairs.Count(SOSbytes) >= 6) { - _jpgs.Add(fullPath.NormalizePath()/* + ' ' + bytePairs.Count(DCTbytes) + ' ' + bytePairs.Count(SOSbytes)*/); + _jpgs.Add(fullPath.NormalizePath()); } } @@ -153,7 +165,7 @@ private void VerifyJpg(string fullPath, Stream stream) WaveFormat fmt = reader.WaveFormat; if (fmt.Encoding != WaveFormatEncoding.Pcm) - return $"has invalid encoding {fmt.Encoding} instead of PCM"; + return $"invalid encoding {fmt.Encoding} instead of PCM"; List errors = []; @@ -164,10 +176,10 @@ private void VerifyJpg(string fullPath, Stream stream) errors.Add($"expected 16bit instead of {fmt.BitsPerSample}bit"); if (fmt.SampleRate is not 44100 and not 44100 / 2 and not 44100 / 4) - errors.Add($"expected multiple of 44.1 kHz instead of {fmt.SampleRate}"); + errors.Add($"expected 44.1 kHz supported sample rate instead of {fmt.SampleRate}"); if (errors.Count > 0) - return $"has invalid audio format: {string.Join(" | ", errors)}"; + return $"invalid audio format: {string.Join(" | ", errors)}"; } catch (Exception e) { From 6b63d55e43714800cdf1ad037513ebef2335e9cb Mon Sep 17 00:00:00 2001 From: ovska Date: Sun, 27 Oct 2024 15:03:28 +0200 Subject: [PATCH 02/12] feat: rename mapname_levelshots.shader on rename --- Pack3r.Core/Parsers/ShaderParser.cs | 2 ++ Pack3r.Core/Services/AssetService.cs | 23 ++++++++++++++--------- Pack3r.Core/Services/Packager.cs | 13 +++++++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Pack3r.Core/Parsers/ShaderParser.cs b/Pack3r.Core/Parsers/ShaderParser.cs index d28e630..c3718d8 100644 --- a/Pack3r.Core/Parsers/ShaderParser.cs +++ b/Pack3r.Core/Parsers/ShaderParser.cs @@ -209,7 +209,9 @@ void HandleCC(string type) logger.Info($"Packed shader '{name}' on line {shader.Line} in '{shader.Asset.Name}' will be modified to account for --rename"); if (!map.ShaderConvert.TryGetValue(shader.Asset, out var list)) + { map.ShaderConvert[shader.Asset] = list = []; + } list.Add((string line, int index) => { diff --git a/Pack3r.Core/Services/AssetService.cs b/Pack3r.Core/Services/AssetService.cs index 6280eb7..cc67f36 100644 --- a/Pack3r.Core/Services/AssetService.cs +++ b/Pack3r.Core/Services/AssetService.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using System.IO; using Pack3r.Extensions; using Pack3r.Logging; using Pack3r.Models; @@ -155,14 +154,7 @@ public async Task GetPackingData(CancellationToken cancellationToken) logger.Info($"Arena skipped, file not found in '{arena.FullName}'"); } - FileInfo levelshot = new(Path.Combine(map.GetMapRoot(), "levelshots", $"{map.Name}.tga")); - - if (!levelshot.Exists) - { - levelshot = new(Path.ChangeExtension(levelshot.FullName, ".jpg")); - } - - if (levelshot.Exists) + if (TryGetLevelshot(out FileInfo levelshot)) { map.RenamableResources.Enqueue(new() { @@ -196,6 +188,19 @@ public async Task GetPackingData(CancellationToken cancellationToken) await ParseResources(map, cancellationToken); return map; + + bool TryGetLevelshot(out FileInfo levelshot) + { + levelshot = new FileInfo(Path.Combine(map.GetMapRoot(), "levelshots", $"{map.Name}.tga")); + + if (levelshot.Exists) + { + return true; + } + + levelshot = new FileInfo(Path.Combine(map.GetMapRoot(), "levelshots", $"{map.Name}.jpg")); + return levelshot.Exists; + } } private async Task ParseResources(Map map, CancellationToken cancellationToken) diff --git a/Pack3r.Core/Services/Packager.cs b/Pack3r.Core/Services/Packager.cs index 4a223d7..d5386ae 100644 --- a/Pack3r.Core/Services/Packager.cs +++ b/Pack3r.Core/Services/Packager.cs @@ -284,7 +284,12 @@ bool TryAddFileFromSource( if (!shader.Asset.Name.EqualsF(relativePath)) convertList = []; - entry = CreateRenamableShader(archive, asset, renamedName: null, convertList, cancellationToken); + // ugly hack to rename levelshots + string archivePath = shader.Asset.Name.EqualsF($"scripts/levelshots_{map.Name}.shader") + ? $"scripts/levelshots_{options.Rename ?? map.Name}" + : shader.Asset.Name; + + entry = CreateRenamableShader(archive, asset, archivePath, convertList, cancellationToken); } else { @@ -390,13 +395,13 @@ void OnFailedAddFile(bool required, ref DefaultInterpolatedStringHandler handler private static ZipArchiveEntry CreateRenamableShader( ZipArchive archive, IAsset asset, - string? renamedName, + string archivePath, List> convertList, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - ZipArchiveEntry entry = archive.CreateEntry(renamedName ?? asset.Name, CompressionLevel.Optimal); + ZipArchiveEntry entry = archive.CreateEntry(archivePath, CompressionLevel.Optimal); if (convertList.Count == 0) { @@ -493,7 +498,7 @@ private static void AddRenamedStylelightShader( CreateRenamableShader( archive, new FileAsset(rootSource, stylelightShaderFile), - renamedName: $"scripts/q3map2_{rename}.shader", + archivePath: $"scripts/q3map2_{rename}.shader", [ (string line, int _) => { From a93728033b5abafb69c4a1e3429b2b7c610bf7d0 Mon Sep 17 00:00:00 2001 From: ovska Date: Sun, 27 Oct 2024 15:03:58 +0200 Subject: [PATCH 03/12] chore: bump to 1.0.0-rc1 --- Directory.Build.props | 2 +- Pack3r.Console/Properties/launchSettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 593a51d..ff85253 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 0.1.10 + 1.0.0-rc1 net8.0 enable enable diff --git a/Pack3r.Console/Properties/launchSettings.json b/Pack3r.Console/Properties/launchSettings.json index 821be6f..493d8a1 100644 --- a/Pack3r.Console/Properties/launchSettings.json +++ b/Pack3r.Console/Properties/launchSettings.json @@ -18,7 +18,7 @@ }, "unsung": { "commandName": "Project", - "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\unsung.map\" -d -m etjump_stable" + "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\unsung.map\" -d -m etjump_stable -r unsung_t1" }, "noargs": { "commandName": "Project", From 812b7364e2938de7c9cfefb3cb3589a611af17f1 Mon Sep 17 00:00:00 2001 From: ovska Date: Wed, 30 Oct 2024 08:58:58 +0200 Subject: [PATCH 04/12] fix: integritychecker warns about valid wav bitness --- Pack3r.Core/Services/IntegrityChecker.cs | 8 ++++---- Pack3r.Tests/Assets/IntegrityTests.cs | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Pack3r.Core/Services/IntegrityChecker.cs b/Pack3r.Core/Services/IntegrityChecker.cs index 555e7f3..cab8b56 100644 --- a/Pack3r.Core/Services/IntegrityChecker.cs +++ b/Pack3r.Core/Services/IntegrityChecker.cs @@ -169,13 +169,13 @@ private void VerifyJpg(string fullPath, Stream stream, bool useAsync) List errors = []; - if (fmt.Channels != 1) + if (fmt.Channels is not 1) errors.Add($"expected mono instead of {fmt.Channels} channels"); - if (fmt.BitsPerSample != 16) - errors.Add($"expected 16bit instead of {fmt.BitsPerSample}bit"); + if (fmt.BitsPerSample is not (8 or 16 or 32)) + errors.Add($"expected 8 or 16 bits per sample instead of {fmt.BitsPerSample}"); - if (fmt.SampleRate is not 44100 and not 44100 / 2 and not 44100 / 4) + if (fmt.SampleRate is not (44100 or 44100 / 2 or 44100 / 4)) errors.Add($"expected 44.1 kHz supported sample rate instead of {fmt.SampleRate}"); if (errors.Count > 0) diff --git a/Pack3r.Tests/Assets/IntegrityTests.cs b/Pack3r.Tests/Assets/IntegrityTests.cs index 6e796e8..b0dab98 100644 --- a/Pack3r.Tests/Assets/IntegrityTests.cs +++ b/Pack3r.Tests/Assets/IntegrityTests.cs @@ -6,7 +6,9 @@ namespace Pack3r.Tests.Assets; public sealed class IntegrityTests { - private readonly IntegrityChecker _checker = new(NullLogger.Instance); + private readonly IntegrityChecker _checker = new( + NullLogger.Instance, + new AppLifetime(NullLogger.Instance, null!)); [Fact] public void Should_Verify_Progressive_Jpg() From 576bda05288f0a5a335e37a5e7e0f6572a26ccef Mon Sep 17 00:00:00 2001 From: ovska Date: Wed, 30 Oct 2024 08:59:44 +0200 Subject: [PATCH 05/12] fix: renamed levelshots shader extension --- Pack3r.Core/Services/Packager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pack3r.Core/Services/Packager.cs b/Pack3r.Core/Services/Packager.cs index d5386ae..027cc27 100644 --- a/Pack3r.Core/Services/Packager.cs +++ b/Pack3r.Core/Services/Packager.cs @@ -286,7 +286,7 @@ bool TryAddFileFromSource( // ugly hack to rename levelshots string archivePath = shader.Asset.Name.EqualsF($"scripts/levelshots_{map.Name}.shader") - ? $"scripts/levelshots_{options.Rename ?? map.Name}" + ? $"scripts/levelshots_{options.Rename ?? map.Name}.shader" : shader.Asset.Name; entry = CreateRenamableShader(archive, asset, archivePath, convertList, cancellationToken); From b4b2fa37b60de82d90223bafea0855d0a705609e Mon Sep 17 00:00:00 2001 From: ovska Date: Wed, 30 Oct 2024 09:00:01 +0200 Subject: [PATCH 06/12] refactor: use QString over ROMchar --- Pack3r.Core/Extensions/ROMCharComparer.cs | 27 ----------------------- Pack3r.Core/Models/IncludedFile.cs | 12 +++++----- Pack3r.Core/Parsers/ShaderParser.cs | 19 ++++++++-------- Pack3r.Core/Parsers/SkinParser.cs | 4 ++-- Pack3r.Core/Services/Packager.cs | 9 ++++---- 5 files changed, 22 insertions(+), 49 deletions(-) delete mode 100644 Pack3r.Core/Extensions/ROMCharComparer.cs diff --git a/Pack3r.Core/Extensions/ROMCharComparer.cs b/Pack3r.Core/Extensions/ROMCharComparer.cs deleted file mode 100644 index 6fa533f..0000000 --- a/Pack3r.Core/Extensions/ROMCharComparer.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Globalization; - -namespace Pack3r.Extensions; - -[Obsolete("Use QString or QPath", true)] -public sealed class ROMCharComparer : IEqualityComparer>, IComparer> -{ - public static readonly ROMCharComparer Instance = new(); - - private ROMCharComparer() { } - - public bool Equals(ReadOnlyMemory x, ReadOnlyMemory y) - { - return x.Span.Equals(y.Span, StringComparison.OrdinalIgnoreCase); - } - - public int GetHashCode([DisallowNull] ReadOnlyMemory obj) - { - return CultureInfo.InvariantCulture.CompareInfo.GetHashCode(obj.Span, CompareOptions.OrdinalIgnoreCase); - } - - public int Compare(ReadOnlyMemory x, ReadOnlyMemory y) - { - return CultureInfo.InvariantCulture.CompareInfo.Compare(x.Span, y.Span, CompareOptions.OrdinalIgnoreCase); - } -} diff --git a/Pack3r.Core/Models/IncludedFile.cs b/Pack3r.Core/Models/IncludedFile.cs index 6834eaa..651b38c 100644 --- a/Pack3r.Core/Models/IncludedFile.cs +++ b/Pack3r.Core/Models/IncludedFile.cs @@ -9,13 +9,13 @@ public class IncludedFile { public AssetSource? Source { get; } public Shader? Shader { get; } - public ReadOnlyMemory SourcePath { get; init; } - public QPath ArchivePath { get; init; } + public QString SourcePath { get; init; } + public QString ArchivePath { get; init; } public bool SourceOnly { get; init; } public IResourceSource? Reference { get; init; } public IncludedFile( - ReadOnlyMemory sourcePath, + QString sourcePath, QPath archivePath) { SourcePath = sourcePath; @@ -24,11 +24,11 @@ public IncludedFile( public IncludedFile(RenamableResource resource) { - SourcePath = resource.AbsolutePath.AsMemory(); - ArchivePath = resource.ArchivePath.AsMemory(); + SourcePath = resource.AbsolutePath; + ArchivePath = resource.ArchivePath; } - public IncludedFile(AssetSource source, ReadOnlyMemory relativePath, Resource resource, Shader? shader = null, bool devResource = false) + public IncludedFile(AssetSource source, QString relativePath, Resource resource, Shader? shader = null, bool devResource = false) { Source = source; SourcePath = Path.Combine(source.RootPath, relativePath.ToString()).NormalizePath().AsMemory(); diff --git a/Pack3r.Core/Parsers/ShaderParser.cs b/Pack3r.Core/Parsers/ShaderParser.cs index c3718d8..7b38aa0 100644 --- a/Pack3r.Core/Parsers/ShaderParser.cs +++ b/Pack3r.Core/Parsers/ShaderParser.cs @@ -358,15 +358,9 @@ public async IAsyncEnumerable Parse( if (line.Value.StartsWithF("implicit")) { - token = line.Value["implicit".Length..]; - - if (!token.TryReadPastWhitespace(out token)) - { - logger.Warn($"Missing implicit mapping path on line {line.Index} in shader '{shader.Name}' in file '{asset.FullPath}'"); - } - else + if (line.Value.TryReadPastWhitespace(out token)) { - if (token.Span.Equals("-", StringComparison.Ordinal)) + if (token.Length == 1 && token.Span.Equals("-", StringComparison.Ordinal)) { shader.ImplicitMapping = shader.Name; } @@ -375,6 +369,10 @@ public async IAsyncEnumerable Parse( shader.ImplicitMapping = token; } } + else + { + logger.Warn($"Missing implicit mapping path on line {line.Index} in shader '{shader.Name}' in file '{asset.FullPath}'"); + } } else if (line.MatchKeyword("skyparms", out token)) { @@ -384,7 +382,7 @@ public async IAsyncEnumerable Parse( continue; } - if (token.Span.Equals("-", StringComparison.Ordinal)) + if (token.Span.Length == 1 && token.Span.Equals("-", StringComparison.Ordinal)) { token = shader.Name; } @@ -473,6 +471,9 @@ public async IAsyncEnumerable Parse( } } + /// + /// Returns whether the line is not empty (just a single opening brace) + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsContinuationBrace(ref Line line) { diff --git a/Pack3r.Core/Parsers/SkinParser.cs b/Pack3r.Core/Parsers/SkinParser.cs index 415662b..b3e0853 100644 --- a/Pack3r.Core/Parsers/SkinParser.cs +++ b/Pack3r.Core/Parsers/SkinParser.cs @@ -20,8 +20,8 @@ public class SkinParser(ILineReader reader) : IReferenceParser if (comma >= 0) { - result.Add( - new Resource(line.Value[(comma + 1)..].Trim().Trim('"'), isShader: true, in line)); + var value = line.Value[(comma + 1)..].Trim().Trim('"'); + result.Add(new Resource(value, isShader: true, in line)); } } diff --git a/Pack3r.Core/Services/Packager.cs b/Pack3r.Core/Services/Packager.cs index 027cc27..74def64 100644 --- a/Pack3r.Core/Services/Packager.cs +++ b/Pack3r.Core/Services/Packager.cs @@ -1,6 +1,4 @@ -using System.Diagnostics; -using System.IO; -using System.IO.Compression; +using System.IO.Compression; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -62,6 +60,7 @@ public async Task CreateZip( foreach (var res in renamable) { CreateRenamable(archive, options, res, cancellationToken); + handledFiles.Add(res.ArchivePath); progress.Report(++i); includedFiles.Add(new IncludedFile(res)); @@ -316,7 +315,7 @@ bool TryAddFileFromSource( { includedFiles.Add(new IncludedFile( source, - asset != null ? asset.Name.AsMemory() : relativePath, + asset != null ? (QString)asset.Name : relativePath, resource, shader, devResource)); @@ -354,7 +353,7 @@ bool TryAddFileAbsolute(string archivePath, string absolutePath) { archive.CreateEntryFromFile(absolutePath, archivePath); handledFiles.Add(archivePath.AsMemory()); - includedFiles.Add(new IncludedFile(sourcePath: absolutePath.AsMemory(), archivePath: archivePath.AsMemory())); + includedFiles.Add(new IncludedFile(sourcePath: absolutePath, archivePath: archivePath)); return true; } catch (IOException ioex) { ex = ioex; } From 19b553cf526c8cdd1412c117470a142dec989f85 Mon Sep 17 00:00:00 2001 From: ovska Date: Fri, 1 Nov 2024 15:14:18 +0200 Subject: [PATCH 07/12] unify ROM/QString usage --- Pack3r.Core/Extensions/StringExtensions.cs | 71 ++++++++-------------- Pack3r.Core/Models/Resource.cs | 4 +- Pack3r.Core/Parsers/ShaderParser.cs | 16 ++++- Pack3r.Core/QString.cs | 2 + Pack3r.Tests/ExtensionTests/MemoryTests.cs | 21 ++++++- 5 files changed, 61 insertions(+), 53 deletions(-) diff --git a/Pack3r.Core/Extensions/StringExtensions.cs b/Pack3r.Core/Extensions/StringExtensions.cs index f106cd2..2eae5c7 100644 --- a/Pack3r.Core/Extensions/StringExtensions.cs +++ b/Pack3r.Core/Extensions/StringExtensions.cs @@ -11,19 +11,23 @@ public static class StringExtensions { public static string NormalizePath(this string path) => path.Replace(Path.DirectorySeparatorChar, '/'); - public static ReadOnlyMemory ChangeExtension(this ReadOnlyMemory file, ReadOnlySpan extension) + public static QString ChangeExtension(this ReadOnlyMemory file, ReadOnlySpan extension) { - int extensionLength = file.GetExtension().Length; - - var withoutExtension = file[..^extensionLength]; + ReadOnlySpan current = file.GetExtension(); if (extension.IsEmpty) - return withoutExtension; + { + return file[..^current.Length]; + } - return $"{withoutExtension.Span}{extension}".AsMemory(); + if (current.SequenceEqual(extension)) + { + return file; + } + + return $"{file[..^current.Length]}{extension}".AsMemory(); } - public static TextureExtension GetTextureExtension(this ReadOnlyMemory path) => GetTextureExtension(path.Span); public static TextureExtension GetTextureExtension(this string path) => GetTextureExtension(path.AsSpan()); public static TextureExtension GetTextureExtension(this ReadOnlySpan path) { @@ -41,29 +45,9 @@ public static TextureExtension GetTextureExtension(this ReadOnlySpan path) return TextureExtension.Other; } - public static ArraySegment Split( - this ReadOnlyMemory value, - char separator, - StringSplitOptions options = StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - { - var ranges = new Range[32]; - - int count = value.Span.Split(ranges.AsSpan(), separator, options); - - if (count == 0) - { - ranges[0] = Range.All; - count = 1; - } - - return new ArraySegment(ranges, 0, count); - } - public static bool TryReadUpToWhitespace(in this ReadOnlyMemory value, out ReadOnlyMemory token) { - var span = value.Span; - - int space = span.IndexOfAny(Tokens.SpaceOrTab); + int space = value.Span.IndexOfAny(Tokens.SpaceOrTab); if (space == -1) { @@ -77,9 +61,7 @@ public static bool TryReadUpToWhitespace(in this ReadOnlyMemory value, out public static bool TryReadPastWhitespace(in this ReadOnlyMemory value, out ReadOnlyMemory token) { - var span = value.Span; - - int space = span.IndexOfAny(Tokens.SpaceOrTab); + int space = value.Span.IndexOfAny(Tokens.SpaceOrTab); if (space == -1) { @@ -111,30 +93,25 @@ public static bool MatchKeyword(in this Line line, string prefix, out ReadOnlyMe public static ReadOnlyMemory TrimQuotes(this ReadOnlyMemory token) { - var keySpan = token.Span; - - if (keySpan.Length >= 2 && - keySpan[0] == '"' && - keySpan[^1] == '"') - { - return token[1..^1]; - } - + _ = TryTrimQuotes(token, out token); return token; } public static bool TryTrimQuotes(this ReadOnlyMemory token, out ReadOnlyMemory trimmed) { - var keySpan = token.Span; - - if (keySpan.Length >= 2 && - keySpan[0] == '"' && - keySpan[^1] == '"') + if (token.Length >= 2) { - trimmed = token[1..^1]; - return true; + var keySpan = token.Span; + + if (keySpan[0] == '"' && + keySpan[^1] == '"') + { + trimmed = token[1..^1]; + return true; + } } + trimmed = token; return false; } diff --git a/Pack3r.Core/Models/Resource.cs b/Pack3r.Core/Models/Resource.cs index 43aff29..537dd89 100644 --- a/Pack3r.Core/Models/Resource.cs +++ b/Pack3r.Core/Models/Resource.cs @@ -10,10 +10,10 @@ namespace Pack3r.Models; public sealed class Resource : IEquatable { public static Resource Shader(string value, in Line line) => new(value.AsMemory(), true, in line); - public static Resource Shader(ReadOnlyMemory value, in Line line) => new(value, true, in line); + public static Resource Shader(QString value, in Line line) => new(value, true, in line); public static Resource File(string value, in Line line) => new(value.AsMemory(), false, in line); - public static Resource File(ReadOnlyMemory value, in Line line) => new(value, false, in line); + public static Resource File(QString value, in Line line) => new(value, false, in line); public static Resource FromModel(QPath value, IResourceSource source) => new(value, source); diff --git a/Pack3r.Core/Parsers/ShaderParser.cs b/Pack3r.Core/Parsers/ShaderParser.cs index 7b38aa0..887ba7d 100644 --- a/Pack3r.Core/Parsers/ShaderParser.cs +++ b/Pack3r.Core/Parsers/ShaderParser.cs @@ -448,8 +448,20 @@ public async IAsyncEnumerable Parse( // read past the frames-agument if (token.TryReadPastWhitespace(out token)) { - foreach (var range in token.Split(' ')) - shader.Resources.Add(token[range]); + AddFrames(shader.Resources, token); + + static void AddFrames(List list, ReadOnlyMemory source) + { + foreach (var match in Tokens.WhitespaceSeparatedTokens().EnumerateMatches(source.Span)) + { + ReadOnlyMemory value = source.Slice(match).TrimQuotes().Trim(); + + if (!value.IsEmpty) + { + list.Add(value); + } + } + } } else { diff --git a/Pack3r.Core/QString.cs b/Pack3r.Core/QString.cs index fa310f4..26ec00d 100644 --- a/Pack3r.Core/QString.cs +++ b/Pack3r.Core/QString.cs @@ -15,6 +15,8 @@ namespace Pack3r; public ReadOnlyMemory Value { get; } public ReadOnlySpan Span => Value.Span; + public bool IsEmpty => Value.IsEmpty; + public char this[int index] => Span[index]; public QString this[Range range] => new(Value, range); diff --git a/Pack3r.Tests/ExtensionTests/MemoryTests.cs b/Pack3r.Tests/ExtensionTests/MemoryTests.cs index ea60d78..96a0f38 100644 --- a/Pack3r.Tests/ExtensionTests/MemoryTests.cs +++ b/Pack3r.Tests/ExtensionTests/MemoryTests.cs @@ -1,5 +1,7 @@ -using Pack3r.Extensions; +using System.Collections.Generic; +using Pack3r.Extensions; using Pack3r.IO; +using Pack3r.Models; namespace Pack3r.Tests.ExtensionTests; @@ -54,9 +56,24 @@ public static void Should_Read_Up_To_Whitespace(string input, string expected) [InlineData("test arg", (string[])["test", "arg"])] [InlineData("test arg arg2", (string[])["test", "arg", "arg2"])] [InlineData("test arg ", (string[])["test", "arg"])] + [InlineData("test \"arg\" ", (string[])["test", "arg"])] + [InlineData("test\t\t\"arg\" ", (string[])["test", "arg"])] + [InlineData("\t\"test\" \"arg\" ", (string[])["test", "arg"])] public static void Should_Split_By_Whitespace(string input, string[] expected) { - Assert.Equal(expected, input.AsMemory().Split(' ').Select(r => input.AsMemory(r).ToString())); + List list = []; + + foreach (var frame in Tokens.WhitespaceSeparatedTokens().EnumerateMatches(input)) + { + ReadOnlyMemory value = input.AsMemory(frame.Index, frame.Length).TrimQuotes().Trim(); + + if (!value.IsEmpty) + { + list.Add(value.ToString()); + } + } + + Assert.Equal(expected, list); } [Theory] From 42c1c294c6ce89b387fd8983a8fd5247ac384073 Mon Sep 17 00:00:00 2001 From: ovska Date: Fri, 1 Nov 2024 15:14:33 +0200 Subject: [PATCH 08/12] add common mapping assets to exclude --- Pack3r.Console/RootCommand.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Pack3r.Console/RootCommand.cs b/Pack3r.Console/RootCommand.cs index 0906030..8f0e108 100644 --- a/Pack3r.Console/RootCommand.cs +++ b/Pack3r.Console/RootCommand.cs @@ -80,7 +80,15 @@ public class RootCommand Arity = CliArgumentArity.ZeroOrMore, ValidationRules = CliValidationRules.LegalPath, AllowMultipleArgumentsPerToken = true)] - public List Exclude { get; set; } = ["pak0.pk3", "pak0.pk3dir"]; + public List Exclude { get; set; } = + [ + "pak0.pk3", + "pak0.pk3dir", + "lights.pk3", + "sd-mapobjects.pk3", + "common.pk3", + "astro-skies.pk3", + ]; [CliOption( Description = "Adds all pk3s in mod directories to exclude-list", From 52e2f8d6d0eac25220e1de0535698bbf76e9ace7 Mon Sep 17 00:00:00 2001 From: ovska Date: Fri, 1 Nov 2024 20:33:22 +0200 Subject: [PATCH 09/12] feat: rename options --- Pack3r.Console/RootCommand.cs | 37 +++++----- Pack3r.Core/IO/AssetSource.cs | 6 +- Pack3r.Core/IO/DirectoryAssetSource.cs | 4 +- Pack3r.Core/IO/Pk3AssetSource.cs | 4 +- Pack3r.Core/Models/Map.cs | 28 ++++---- Pack3r.Core/PackOptions.cs | 6 +- Pack3r.Core/Parsers/MapFileParser.cs | 2 +- Pack3r.Core/Parsers/ResourceRefParser.cs | 2 +- Pack3r.Core/Parsers/ShaderParser.cs | 8 +-- Pack3r.Core/Services/AssetService.cs | 88 +++++++++++++----------- Pack3r.Core/Services/Packager.cs | 8 +-- Pack3r.Tests/MapParserTests.cs | 2 +- Pack3r.Tests/ShaderParserTests.cs | 2 +- 13 files changed, 102 insertions(+), 95 deletions(-) diff --git a/Pack3r.Console/RootCommand.cs b/Pack3r.Console/RootCommand.cs index 8f0e108..aa79277 100644 --- a/Pack3r.Console/RootCommand.cs +++ b/Pack3r.Console/RootCommand.cs @@ -19,7 +19,7 @@ public class RootCommand public FileInfo Map { get; set; } = null!; [CliOption( - Description = "Path to destination pk3 or directory, defaults to etmain", + Description = "Path to destination pk3/zip (or directory where it will be written), defaults to etmain/mapname.pk3", Required = false, ValidationRules = CliValidationRules.LegalPath)] public FileSystemInfo? Output { get; set; } @@ -28,7 +28,7 @@ public class RootCommand public bool DryRun { get; set; } [CliOption( - Description = "Map release name (bsp, lightmaps, mapscript, etc.)", + Description = "Name of the map after packing (renames bsp, lightmaps, mapscript etc.)", Required = false, ValidationRules = CliValidationRules.LegalFileName)] public string? Rename { get; set; } @@ -42,7 +42,7 @@ public class RootCommand public bool Loose { get; set; } [CliOption( - Description = "Pack source files such as .map, editorimages, misc_models")] + Description = "Pack only source files (.map, editorimages, misc_models) without packing BSP & lightmaps")] public bool Source { get; set; } [CliOption( @@ -61,26 +61,30 @@ public class RootCommand public bool ReferenceDebug { get; set; } [CliOption( - Description = "Overwrite existing files in the output path with impunity")] + Description = "Overwrite existing files in the output path with impunity", + Aliases = ["-f", "--overwrite"])] public bool Force { get; set; } [CliOption( - Description = "Include pk3 files and pk3dirs in etmain when indexing files")] + Description = "Include pk3 files and pk3dirs in etmain when indexing files", + Aliases = ["-pk3"])] public bool IncludePk3 { get; set; } [CliOption( - Description = "Ignore some pk3 files or pk3dir directories", + Description = "Don't scan pk3/pk3dirs for assets", Arity = CliArgumentArity.ZeroOrMore, ValidationRules = CliValidationRules.LegalPath, - AllowMultipleArgumentsPerToken = true)] - public List Ignore { get; set; } = ["pak1.pk3", "pak2.pk3", "mp_bin.pk3"]; + AllowMultipleArgumentsPerToken = true, + Aliases = ["-ns"])] + public List NoScan { get; set; } = ["pak1.pk3", "pak2.pk3", "mp_bin.pk3"]; [CliOption( - Description = "Never pack files found in these pk3s or directories", + Description = "Scan some pk3s/pk3dirs but don't pack their contants", Arity = CliArgumentArity.ZeroOrMore, ValidationRules = CliValidationRules.LegalPath, - AllowMultipleArgumentsPerToken = true)] - public List Exclude { get; set; } = + AllowMultipleArgumentsPerToken = true, + Aliases = ["-np"])] + public List NoPack { get; set; } = [ "pak0.pk3", "pak0.pk3dir", @@ -104,11 +108,11 @@ public Task RunAsync() MapFile = ResolveMap(), Pk3File = ResolvePk3(), Overwrite = Force, - ExcludeSources = Exclude, - IgnoreSources = Ignore, + UnpackedSources = NoPack, + UnscannedSources = NoScan, ModFolders = Mods, DryRun = DryRun, - IncludeSource = Source, + OnlySource = Source, LoadPk3s = IncludePk3, LogLevel = Verbosity, Rename = Rename, @@ -116,7 +120,7 @@ public Task RunAsync() UseShaderlist = Shaderlist, ShaderDebug = ShaderDebug, ReferenceDebug = ReferenceDebug, - }); + }); ; ; } private FileInfo ResolveMap() @@ -158,7 +162,8 @@ private FileInfo ResolveMap() pk3Location = etmain; } - pk3 = new FileInfo(Path.Combine(pk3Location.FullName, Path.ChangeExtension(pk3Name, ".pk3"))); + string outName = Path.ChangeExtension(pk3Name, Source ? ".zip" : ".pk3"); + pk3 = new FileInfo(Path.Combine(pk3Location.FullName, outName)); } else { diff --git a/Pack3r.Core/IO/AssetSource.cs b/Pack3r.Core/IO/AssetSource.cs index 5aa8d0b..69c7080 100644 --- a/Pack3r.Core/IO/AssetSource.cs +++ b/Pack3r.Core/IO/AssetSource.cs @@ -18,7 +18,7 @@ public abstract IAsyncEnumerable EnumerateShaders( /// /// Whether this source is used to discover files, but never pack them (pak0, mod files etc). /// - public bool IsExcluded { get; } + public bool NotPacked { get; } /// /// Display name (folder/pk3 name). @@ -31,9 +31,9 @@ public abstract IAsyncEnumerable EnumerateShaders( protected bool _disposed; - protected AssetSource(bool isExcluded) + protected AssetSource(bool notPacked) { - IsExcluded = isExcluded; + NotPacked = notPacked; _assetsLazy = new(InitializeAssets, LazyThreadSafetyMode.ExecutionAndPublication); } diff --git a/Pack3r.Core/IO/DirectoryAssetSource.cs b/Pack3r.Core/IO/DirectoryAssetSource.cs index d9293d6..65d92e3 100644 --- a/Pack3r.Core/IO/DirectoryAssetSource.cs +++ b/Pack3r.Core/IO/DirectoryAssetSource.cs @@ -6,13 +6,13 @@ namespace Pack3r.IO; [DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class DirectoryAssetSource(DirectoryInfo directory, bool isExcluded) : AssetSource(isExcluded) +public sealed class DirectoryAssetSource(DirectoryInfo directory, bool notPacked) : AssetSource(notPacked) { public DirectoryInfo Directory => directory; public override string RootPath => directory.FullName; public override string ToString() => $"{{ Dir: {Directory.FullName} }}"; - internal string DebuggerDisplay => $"{{ Dir src: '{Directory.Name}' (Excluded: {IsExcluded}) }}"; + internal string DebuggerDisplay => $"{{ Dir src: '{Directory.Name}' (Excluded: {NotPacked}) }}"; public override async IAsyncEnumerable EnumerateShaders( IShaderParser parser, diff --git a/Pack3r.Core/IO/Pk3AssetSource.cs b/Pack3r.Core/IO/Pk3AssetSource.cs index 078f7b0..773378d 100644 --- a/Pack3r.Core/IO/Pk3AssetSource.cs +++ b/Pack3r.Core/IO/Pk3AssetSource.cs @@ -8,7 +8,7 @@ namespace Pack3r.IO; [DebuggerDisplay("{DebuggerDisplay,nq}")] -public sealed class Pk3AssetSource(string path, bool isExcluded) : AssetSource(isExcluded) +public sealed class Pk3AssetSource(string path, bool notPacked) : AssetSource(notPacked) { public string ArchivePath => path; public override string RootPath => ArchivePath; @@ -16,7 +16,7 @@ public sealed class Pk3AssetSource(string path, bool isExcluded) : AssetSource(i private readonly ZipArchive _archive = ZipFile.OpenRead(path); public override string ToString() => $"{{ Pk3: {ArchivePath} }}"; - internal string DebuggerDisplay => $"{{ Pk3 src: '{Path.GetFileName(ArchivePath)}' (Excluded: {IsExcluded}) }}"; + internal string DebuggerDisplay => $"{{ Pk3 src: '{Path.GetFileName(ArchivePath)}' (Excluded: {NotPacked}) }}"; public override async IAsyncEnumerable EnumerateShaders( IShaderParser parser, diff --git a/Pack3r.Core/Models/Map.cs b/Pack3r.Core/Models/Map.cs index c562094..fd87e8a 100644 --- a/Pack3r.Core/Models/Map.cs +++ b/Pack3r.Core/Models/Map.cs @@ -106,7 +106,7 @@ 3. pk3dirs in reverse alphabetical order .EnumerateDirectories("*.pk3dir", SearchOption.TopDirectoryOnly) .OrderByDescending(dir => dir.Name, StringComparer.OrdinalIgnoreCase)) { - if (IsExcluded(pk3dir) == SourceFilter.Ignored) + if (GetSourceFilter(pk3dir) == SourceFilter.Ignored) continue; if (unique.Add(pk3dir.FullName)) @@ -122,7 +122,7 @@ private ImmutableArray InitAssetSources() foreach (var dir in AssetDirectories) { - var dirFilter = IsExcluded(dir); + var dirFilter = GetSourceFilter(dir); // never ignore etmain if (dirFilter == SourceFilter.Ignored && !dir.FullName.EqualsF(ETMain.FullName)) @@ -130,25 +130,25 @@ private ImmutableArray InitAssetSources() continue; } - list.Add(new DirectoryAssetSource(dir, isExcluded: dirFilter == SourceFilter.Excluded)); + list.Add(new DirectoryAssetSource(dir, notPacked: dirFilter == SourceFilter.Excluded)); // pk3s need to be loaded always if there are pk3/dir excludes - if (_options.LoadPk3s || _options.ExcludeSources.Count > 0) + if (_options.LoadPk3s || _options.UnpackedSources.Count > 0) { foreach (var file in dir.EnumerateFiles("*.pk3", SearchOption.TopDirectoryOnly)) { - var pk3Filter = IsExcluded(file); + var pk3Filter = GetSourceFilter(file); if (pk3Filter == SourceFilter.Ignored) continue; // exclude all pk3s in an excluded pk3dir - bool pk3isExcluded = dir.FullName.HasExtension("pk3dir") + bool notPacked = dir.FullName.HasExtension("pk3dir") ? (dirFilter == SourceFilter.Excluded || pk3Filter == SourceFilter.Excluded) : pk3Filter == SourceFilter.Excluded; - if (_options.LoadPk3s || pk3isExcluded) - list.Add(new Pk3AssetSource(file.FullName, isExcluded: pk3isExcluded)); + if (_options.LoadPk3s || notPacked) + list.Add(new Pk3AssetSource(file.FullName, notPacked: notPacked)); } } } @@ -162,11 +162,9 @@ private ImmutableArray InitAssetSources() { if (mod.EqualsF(dir.Name)) { - //list.Add(new DirectoryAssetSource(dir, isExcluded: true)); - foreach (var file in dir.EnumerateFiles("*.pk3", SearchOption.TopDirectoryOnly)) { - list.Add(new Pk3AssetSource(file.FullName, isExcluded: true)); + list.Add(new Pk3AssetSource(file.FullName, notPacked: true)); } break; @@ -184,24 +182,24 @@ 1. pak0 is always first (and other excluded sources) byte[] sortKeys = new byte[512]; return list - .OrderByDescending(s => (s.IsExcluded, s.Name.EqualsF("pak0.pk3"))) // pak0 first, then other excludes + .OrderByDescending(s => (s.NotPacked, s.Name.EqualsF("pak0.pk3"))) // pak0 first, then other excludes .ThenBy(s => s is DirectoryAssetSource d ? AssetDirectories.IndexOf(d.Directory) : int.MaxValue) .ThenByDescending(s => IOPath.GetFileNameWithoutExtension(s.RootPath)) .ToImmutableArray(); #pragma warning restore IDE0305 // Simplify collection initialization } - private SourceFilter IsExcluded(FileSystemInfo item) + private SourceFilter GetSourceFilter(FileSystemInfo item) { ReadOnlySpan dirOrPk3 = IOPath.GetFileName(item.FullName.AsSpan()); - foreach (var value in _options.IgnoreSources) + foreach (var value in _options.UnscannedSources) { if (dirOrPk3.EqualsF(value)) return SourceFilter.Ignored; } - foreach (var value in _options.ExcludeSources) + foreach (var value in _options.UnpackedSources) { if (dirOrPk3.EqualsF(value)) return SourceFilter.Excluded; diff --git a/Pack3r.Core/PackOptions.cs b/Pack3r.Core/PackOptions.cs index 07ef3a8..da5bde2 100644 --- a/Pack3r.Core/PackOptions.cs +++ b/Pack3r.Core/PackOptions.cs @@ -17,7 +17,7 @@ public class PackOptions public bool ReferenceDebug { get; set; } - public bool IncludeSource { get; set; } + public bool OnlySource { get; set; } public bool RequireAllAssets { get; set; } @@ -29,9 +29,9 @@ public class PackOptions public bool LoadPk3s { get; set; } - public List IgnoreSources { get; init; } = null!; + public List UnscannedSources { get; init; } = null!; - public List ExcludeSources { get; init; } = null!; + public List UnpackedSources { get; init; } = null!; public List ModFolders { get; init; } = null!; } diff --git a/Pack3r.Core/Parsers/MapFileParser.cs b/Pack3r.Core/Parsers/MapFileParser.cs index 5eb3d7b..e7e4e3b 100644 --- a/Pack3r.Core/Parsers/MapFileParser.cs +++ b/Pack3r.Core/Parsers/MapFileParser.cs @@ -213,7 +213,7 @@ void HandleKeysAndClear() bool isMiscModel = IsClassName("misc_model"); var res = new Resource(value, isShader: false, in line, sourceOnly: isMiscModel); - if (options.IncludeSource || !isMiscModel) + if (options.OnlySource || !isMiscModel) { referenceResources.Add(res); resources.Add(res); diff --git a/Pack3r.Core/Parsers/ResourceRefParser.cs b/Pack3r.Core/Parsers/ResourceRefParser.cs index 7d83af8..836ae07 100644 --- a/Pack3r.Core/Parsers/ResourceRefParser.cs +++ b/Pack3r.Core/Parsers/ResourceRefParser.cs @@ -68,7 +68,7 @@ public async Task ParseReferences( { result.Remove(item); - if (options.IncludeSource) + if (options.OnlySource) { result.Add(new Resource( item.Value, diff --git a/Pack3r.Core/Parsers/ShaderParser.cs b/Pack3r.Core/Parsers/ShaderParser.cs index 887ba7d..57229db 100644 --- a/Pack3r.Core/Parsers/ShaderParser.cs +++ b/Pack3r.Core/Parsers/ShaderParser.cs @@ -103,12 +103,12 @@ bool skipPredicate(string name) } // cases such as common shaders, if they are compile only we don't care - if (!options.IncludeSource && (!a.NeededInPk3 || !b.NeededInPk3)) + if (!options.OnlySource && (!a.NeededInPk3 || !b.NeededInPk3)) { return a.NeededInPk3 ? b : a; } - if (!a.Source.IsExcluded || !b.Source.IsExcluded) + if (!a.Source.NotPacked || !b.Source.NotPacked) { duplicate.AddOrUpdate( key: a.Name, @@ -336,7 +336,7 @@ public async IAsyncEnumerable Parse( } } - if (!found && options.IncludeSource) + if (!found && options.OnlySource) { foreach (var prefix in _devTexturePrefixes) { @@ -563,7 +563,7 @@ private bool CanSkipShaderDirective(ReadOnlySpan line) return (line[0] | 0x20) switch { - 'q' => !options.IncludeSource && line.StartsWithF("qer_"), + 'q' => !options.OnlySource && line.StartsWithF("qer_"), 's' => line.StartsWithF("surfaceparm") || line.StartsWithF("sort"), 'c' => line.StartsWithF("cull"), 'n' => line.StartsWithF("nopicmip") || line.StartsWithF("nomipmaps"), diff --git a/Pack3r.Core/Services/AssetService.cs b/Pack3r.Core/Services/AssetService.cs index cc67f36..a0c5cac 100644 --- a/Pack3r.Core/Services/AssetService.cs +++ b/Pack3r.Core/Services/AssetService.cs @@ -51,7 +51,7 @@ public async Task GetPackingData(CancellationToken cancellationToken) { string srcMsg = string.Join( Environment.NewLine, - map.AssetSources.Select(src => $"\t{src.RootPath}{(src.IsExcluded ? " (not packed)" : "")}")); + map.AssetSources.Select(src => $"\t{src.RootPath}{(src.NotPacked ? " (not packed)" : "")}")); logger.System($"Using sources for discovery: {Environment.NewLine}{srcMsg}"); } else @@ -61,51 +61,52 @@ public async Task GetPackingData(CancellationToken cancellationToken) logger.System($"Using directories{pk3msg} for discovery: {dirMsg}"); } - // add bsp - FileInfo bsp = new(Path.ChangeExtension(map.Path, "bsp")); - map.RenamableResources.Enqueue(new() - { - Name = "bsp", - AbsolutePath = bsp.FullName, - ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}.bsp") - }); - - // add .map - if (options.IncludeSource) + if (!options.OnlySource) { + // add bsp + FileInfo bsp = new(Path.ChangeExtension(map.Path, "bsp")); map.RenamableResources.Enqueue(new() { - Name = "map source", - AbsolutePath = map.Path, - ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}.map"), + Name = "bsp", + AbsolutePath = bsp.FullName, + ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}.bsp") }); - } - - var lightmapDir = new DirectoryInfo(Path.ChangeExtension(map.Path, null)); - if (lightmapDir.Exists && lightmapDir.GetFiles("lm_????.tga") is { Length: > 0 } lmFiles) - { - map.HasLightmaps = true; - bool timestampWarned = false; + var lightmapDir = new DirectoryInfo(Path.ChangeExtension(map.Path, null)); - for (int i = 0; i < lmFiles.Length; i++) + if (lightmapDir.Exists && lightmapDir.GetFiles("lm_????.tga") is { Length: > 0 } lmFiles) { - FileInfo? file = lmFiles[i]; - timestampWarned = timestampWarned || logger.CheckAndLogTimestampWarning("Lightmaps", bsp, file); + map.HasLightmaps = true; + bool timestampWarned = false; - map.RenamableResources.Enqueue(new() + for (int i = 0; i < lmFiles.Length; i++) { - Name = "lightmaps", - AbsolutePath = file.FullName, - ArchivePath = Path.Combine("maps", options.Rename ?? map.Name, file.Name) - }); + FileInfo? file = lmFiles[i]; + timestampWarned = timestampWarned || logger.CheckAndLogTimestampWarning("Lightmaps", bsp, file); + + map.RenamableResources.Enqueue(new() + { + Name = "lightmaps", + AbsolutePath = file.FullName, + ArchivePath = Path.Combine("maps", options.Rename ?? map.Name, file.Name) + }); + } + } + else + { + logger.Info($"Lightmaps skipped, files not found in '{lightmapDir.FullName}'"); } } else { - logger.Info($"Lightmaps skipped, files not found in '{lightmapDir.FullName}'"); + // add .map + map.RenamableResources.Enqueue(new() + { + Name = "map source", + AbsolutePath = map.Path, + ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}.map"), + }); } - var objdata = new FileInfo(Path.ChangeExtension(map.Path, "objdata")); if (objdata.Exists) @@ -171,18 +172,21 @@ public async Task GetPackingData(CancellationToken cancellationToken) $"Levelshot skipped, file not found in '{Path.GetFileNameWithoutExtension(levelshot.FullName.AsSpan())}.tga/.jpg'"); } - if (FindFileFromMods(map, Path.Combine("maps", $"{map.Name}_tracemap.tga")) is { Exists: true } tracemap) + if (!options.OnlySource) { - map.RenamableResources.Enqueue(new() + if (FindFileFromMods(map, Path.Combine("maps", $"{map.Name}_tracemap.tga")) is { Exists: true } tracemap) { - Name = "tracemap", - AbsolutePath = tracemap.FullName, - ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}_tracemap.tga") - }); - } - else - { - logger.Trace($"Tracemap skipped, not found in any directory."); + map.RenamableResources.Enqueue(new() + { + Name = "tracemap", + AbsolutePath = tracemap.FullName, + ArchivePath = Path.Combine("maps", $"{options.Rename ?? map.Name}_tracemap.tga") + }); + } + else + { + logger.Trace($"Tracemap skipped, not found in any directory."); + } } await ParseResources(map, cancellationToken); diff --git a/Pack3r.Core/Services/Packager.cs b/Pack3r.Core/Services/Packager.cs index 74def64..c0de6eb 100644 --- a/Pack3r.Core/Services/Packager.cs +++ b/Pack3r.Core/Services/Packager.cs @@ -215,7 +215,7 @@ bool IsHandledOrExcluded(QPath relativePath) foreach (var source in map.AssetSources) { - if (source.IsExcluded && source.Assets.ContainsKey(relativePath)) + if (source.NotPacked && source.Assets.ContainsKey(relativePath)) return true; } @@ -234,7 +234,7 @@ void AddCompileFile(string absolutePath) void AddShaderFile(Shader shader, Resource resource) { - if (shader.Source.IsExcluded) + if (shader.Source.NotPacked) return; if (TryAddFileFromSource(shader.Source, shader.DestinationPath.AsMemory(), resource, shader)) @@ -251,7 +251,7 @@ void AddFileRelative(QPath relativePath, Resource resource, Shader? shader = nul return; } - if (options.IncludeSource && resource.SourceOnly) + if (options.OnlySource && resource.SourceOnly) devResource = true; string sourceOnly = devResource ? " (source file)" : ""; @@ -295,7 +295,7 @@ bool TryAddFileFromSource( if (!source.Assets.TryGetValue(relativePath, out asset)) return false; - if (source.IsExcluded) + if (source.NotPacked) { // file found from an excluded source entry = null; diff --git a/Pack3r.Tests/MapParserTests.cs b/Pack3r.Tests/MapParserTests.cs index 6dc779e..857e7b4 100644 --- a/Pack3r.Tests/MapParserTests.cs +++ b/Pack3r.Tests/MapParserTests.cs @@ -28,7 +28,7 @@ public static async Task Should_Include_Dev_Assets(bool include) } """); - var map = await CreateParser(reader, new PackOptions { IncludeSource = include, MapFile = null! }) + var map = await CreateParser(reader, new PackOptions { OnlySource = include, MapFile = null! }) .ParseMapAssets("C:/ET/etmain/maps/test.map", default); if (include) diff --git a/Pack3r.Tests/ShaderParserTests.cs b/Pack3r.Tests/ShaderParserTests.cs index bacfb77..90ea9b6 100644 --- a/Pack3r.Tests/ShaderParserTests.cs +++ b/Pack3r.Tests/ShaderParserTests.cs @@ -22,7 +22,7 @@ private static ShaderParser GetParser( var reader = new StringLineReader(data); return new ShaderParser( reader, - new PackOptions { IncludeSource = includeDevFiles, MapFile = null! }, + new PackOptions { OnlySource = includeDevFiles, MapFile = null! }, NullLogger.Instance, new NoOpProgressManager()); } From 34ca106e24a6828a7d66a1e1abdd459656691b39 Mon Sep 17 00:00:00 2001 From: ovska Date: Sat, 2 Nov 2024 21:16:55 +0200 Subject: [PATCH 10/12] feat: aliases --- Pack3r.Console/RootCommand.cs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Pack3r.Console/RootCommand.cs b/Pack3r.Console/RootCommand.cs index aa79277..6702891 100644 --- a/Pack3r.Console/RootCommand.cs +++ b/Pack3r.Console/RootCommand.cs @@ -8,7 +8,8 @@ namespace Pack3r.Console; [CliCommand( Description = "Pack3r, tool to create release-ready pk3s from NetRadiant maps", - NameCasingConvention = CliNameCasingConvention.LowerCase)] + NameCasingConvention = CliNameCasingConvention.LowerCase, + ShortFormAutoGenerate = false)] public class RootCommand { [CliArgument( @@ -21,28 +22,36 @@ public class RootCommand [CliOption( Description = "Path to destination pk3/zip (or directory where it will be written), defaults to etmain/mapname.pk3", Required = false, - ValidationRules = CliValidationRules.LegalPath)] + ValidationRules = CliValidationRules.LegalPath, + Aliases = ["-o"])] public FileSystemInfo? Output { get; set; } - [CliOption(Description = "Discover packed files and estimate file size without creating a pk3")] + [CliOption( + Description = "Discover packed files and estimate file size without creating a pk3", + Aliases = ["-d"])] public bool DryRun { get; set; } [CliOption( Description = "Name of the map after packing (renames bsp, lightmaps, mapscript etc.)", Required = false, - ValidationRules = CliValidationRules.LegalFileName)] + ValidationRules = CliValidationRules.LegalFileName, + Aliases = ["-r"])] public string? Rename { get; set; } [CliOption( Description = "Log severity threshold", - Arity = CliArgumentArity.ZeroOrOne)] + Arity = CliArgumentArity.ZeroOrOne, + Aliases = ["-v"])] public LogLevel Verbosity { get; set; } = LogLevel.Info; - [CliOption(Description = "Complete packing even if some files are missing")] + [CliOption( + Description = "Complete packing even if some files are missing", + Aliases = ["-l"])] public bool Loose { get; set; } [CliOption( - Description = "Pack only source files (.map, editorimages, misc_models) without packing BSP & lightmaps")] + Description = "Pack only source files (.map, editorimages, misc_models) without packing BSP & lightmaps", + Aliases = ["-s"])] public bool Source { get; set; } [CliOption( @@ -67,7 +76,7 @@ public class RootCommand [CliOption( Description = "Include pk3 files and pk3dirs in etmain when indexing files", - Aliases = ["-pk3"])] + Aliases = ["-p", "--pk3"])] public bool IncludePk3 { get; set; } [CliOption( From 2d402c8ecc2f3069201e0912a7eb5929a50ebf50 Mon Sep 17 00:00:00 2001 From: ovska Date: Sat, 2 Nov 2024 21:25:18 +0200 Subject: [PATCH 11/12] remove shaderlist support --- Pack3r.Console/RootCommand.cs | 8 +---- Pack3r.Core/IO/AssetSource.cs | 1 - Pack3r.Core/IO/DirectoryAssetSource.cs | 7 ---- Pack3r.Core/IO/Pk3AssetSource.cs | 16 --------- Pack3r.Core/PackOptions.cs | 2 -- Pack3r.Core/Parsers/ShaderParser.cs | 49 -------------------------- README.md | 1 - 7 files changed, 1 insertion(+), 83 deletions(-) diff --git a/Pack3r.Console/RootCommand.cs b/Pack3r.Console/RootCommand.cs index 6702891..7f86748 100644 --- a/Pack3r.Console/RootCommand.cs +++ b/Pack3r.Console/RootCommand.cs @@ -54,11 +54,6 @@ public class RootCommand Aliases = ["-s"])] public bool Source { get; set; } - [CliOption( - Description = "Only read shaders present in shaderlist.txt if one exists", - Aliases = ["-sl"])] - public bool Shaderlist { get; set; } - [CliOption( Description = "Print shader resolution details (Debug verbosity needed)", Aliases = ["-sd"])] @@ -126,10 +121,9 @@ public Task RunAsync() LogLevel = Verbosity, Rename = Rename, RequireAllAssets = !Loose, - UseShaderlist = Shaderlist, ShaderDebug = ShaderDebug, ReferenceDebug = ReferenceDebug, - }); ; ; + }); } private FileInfo ResolveMap() diff --git a/Pack3r.Core/IO/AssetSource.cs b/Pack3r.Core/IO/AssetSource.cs index 69c7080..4301b4b 100644 --- a/Pack3r.Core/IO/AssetSource.cs +++ b/Pack3r.Core/IO/AssetSource.cs @@ -7,7 +7,6 @@ namespace Pack3r.IO; public abstract class AssetSource : IDisposable { public abstract string RootPath { get; } - public abstract IAsset? GetShaderlist(); public abstract IAsyncEnumerable EnumerateShaders( IShaderParser parser, Func skipPredicate, diff --git a/Pack3r.Core/IO/DirectoryAssetSource.cs b/Pack3r.Core/IO/DirectoryAssetSource.cs index 65d92e3..ed18d0e 100644 --- a/Pack3r.Core/IO/DirectoryAssetSource.cs +++ b/Pack3r.Core/IO/DirectoryAssetSource.cs @@ -41,11 +41,4 @@ protected override IEnumerable EnumerateAssets() .Where(f => Tokens.PackableFile().IsMatch(f.FullName)) .Select(f => new FileAsset(this, f)); } - - public override IAsset? GetShaderlist() - { - return new FileAsset( - this, - new FileInfo(Path.Combine(Directory.FullName, "scripts", "shaderlist.txt"))); - } } diff --git a/Pack3r.Core/IO/Pk3AssetSource.cs b/Pack3r.Core/IO/Pk3AssetSource.cs index 773378d..ca5fcc1 100644 --- a/Pack3r.Core/IO/Pk3AssetSource.cs +++ b/Pack3r.Core/IO/Pk3AssetSource.cs @@ -56,20 +56,4 @@ protected override IEnumerable EnumerateAssets() .Where(entry => entry.FullName.Length < Global.MAX_QPATH && Tokens.PackableFile().IsMatch(entry.FullName.GetExtension())) .Select(entry => new Pk3Asset(this, entry)); } - - public override IAsset? GetShaderlist() - { - ObjectDisposedException.ThrowIf(_disposed, this); - - ZipArchiveEntry? entry = - _archive.GetEntry("scripts/shaderlist.txt") ?? - _archive.GetEntry("scripts\\shaderlist.txt"); - - if (entry is not null) - { - return new Pk3Asset(this, entry); - } - - return null; - } } diff --git a/Pack3r.Core/PackOptions.cs b/Pack3r.Core/PackOptions.cs index da5bde2..c83e498 100644 --- a/Pack3r.Core/PackOptions.cs +++ b/Pack3r.Core/PackOptions.cs @@ -11,8 +11,6 @@ public class PackOptions [MemberNotNullWhen(false, nameof(Pk3File))] public bool DryRun { get; set; } - public bool UseShaderlist { get; set; } - public bool ShaderDebug { get; set; } public bool ReferenceDebug { get; set; } diff --git a/Pack3r.Core/Parsers/ShaderParser.cs b/Pack3r.Core/Parsers/ShaderParser.cs index 57229db..9bc3dea 100644 --- a/Pack3r.Core/Parsers/ShaderParser.cs +++ b/Pack3r.Core/Parsers/ShaderParser.cs @@ -35,30 +35,18 @@ public async Task> GetReferencedShaders( ConcurrentDictionary allShaders = []; ConcurrentDictionary> duplicateShaders = []; - var whitelistsBySource = await ParseShaderLists(map, cancellationToken); int shaderFileCount = 0; using (var progress = progressManager.Create("Parsing shader files", max: null)) { await Parallel.ForEachAsync(map.AssetSources, Global.ParallelOptions(cancellationToken), async (source, ct) => { - var whitelist = whitelistsBySource.GetValueOrDefault(source); - bool skipPredicate(string name) { progress.Report(Interlocked.Increment(ref shaderFileCount)); string fileName = Path.GetFileNameWithoutExtension(name); - if (whitelist is not null && - !fileName.StartsWithF("levelshots") && - !whitelist.Contains(fileName)) - { - if (options.ShaderDebug) - logger.Debug($"Skipped parsing shaders from {getName()} (not in shaderlist)"); - return true; - } - if (fileName.EqualsF("q3shadersCopyForRadiant")) { if (options.ShaderDebug) @@ -498,43 +486,6 @@ private static bool IsContinuationBrace(ref Line line) return false; } - private async Task>> ParseShaderLists( - Map map, - CancellationToken cancellationToken) - { - if (!options.UseShaderlist) - return []; - - var dict = new Dictionary>(); - - foreach (var source in map.AssetSources) - { - var shaderlistAsset = source.GetShaderlist(); - - if (shaderlistAsset is null) - continue; - - try - { - HashSet allowedFiles = []; - - await foreach (var line in reader.ReadLines(shaderlistAsset, default, cancellationToken)) - { - allowedFiles.Add(line.Value); - } - - dict.Add(source, allowedFiles); - } - catch (Exception e) - { - logger.Exception(e, $"Could not read shaderlist '{shaderlistAsset.FullPath}'"); - throw new ControlledException(); - } - } - - return dict; - } - private static readonly ImmutableArray _skySuffixes = [ "_bk", "_dn", "_ft", "_up", "_rt", "_lf" diff --git a/README.md b/README.md index 09e0fba..d77f8fa 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ - `-v, --verbosity` Log severity threshold [default: Info] - `-l, --loose` Complete packing even if some files are missing [default: False] - `-s, --source` Pack source files such as .map, editorimages, misc_models [default: False] -- `-sl, --shaderlist` Only read shaders present in shaderlist.txt [default: False] - `-sd, --shaderdebug` Print shader resolution details (Debug verbosity needed) [default: False] - `-rd, --referencedebug` Print asset resolution details (Info verbosity needed) [default: False] - `-f, --force` Overwrite existing files in the output path with impunity [default: False] From 8bcd6307c2833dfb97011ea6d4b1b3ef58f2798a Mon Sep 17 00:00:00 2001 From: ovska Date: Sat, 2 Nov 2024 21:51:58 +0200 Subject: [PATCH 12/12] update readme --- README.md | 96 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index d77f8fa..9f56a98 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,21 @@ title="Zip icon created by Flat Icons" src="https://github.com/ovska/Pack3r/assets/68028366/5c628e71-bf3f-47e6-9a95-963144fcaa3e" />

Pack3r

-

Create release-ready pk3 archives quickly from .map-files

+

Create release-ready Wolfenstein: Enemy Territory pk3 archives quickly from .map-files

+--- + ## Features -- Parses through your map, shaders, mapscript, etc. and discovers files are needed for the map, while leaving out editorimages, lightimages and other files not needed for release (unless `-s` specified) -- Performs compression in-memory, does not create or leave intermediate files, and never modifies the original files (see below) -- Support for renaming the map to create release versions such as `b1`, with automatic renaming of bsp, mapscript, levelshots, arena and more, while the original files are left untouched (`-r`) -- Extensive logging to trace why each file was included in the pk3, including the exact line in a source file / shader where the file was referenced (`-sd` and `-rd`) +- Parses the map file, shaders, mapscript, etc. and discovers files are needed to play the map. Editorimages, misc_models etc. are left out of the archive (unless wanted) +- Everything is performed in memory without creating any intermediate files, or modifying the originals +- Support for renaming the map to create release versions such as `b1`, with automatic renaming of bsp, mapscript, levelshots, arena and more, while the original files are left untouched +- Extensive logging to trace why each file was included in the pk3, including the exact line/byte offset where in the file the shader/file was referenced. - Support for file discovery from pk3's and pk3dirs, such as `sd-mapobjects.pk3` or a texture/model pack extracted into a separate pk3dir to keep etmain clean +- Warnings about possible pitfalls such as lightmaps from another compile, or image/audio formats not supported by ET 2.60b + +--- ## Usage `Pack3r [options]` @@ -23,33 +28,67 @@ `` Path to the .map file [required] ### Options: -- `-o, --output` Path to destination pk3 or directory, defaults to etmain -- `-d, --dryrun` Discover packed files and estimate file size without creating a pk3 [default: False] -- `-r, --rename` Map release name (bsp, lightmaps, mapscript, etc.) -- `-v, --verbosity` Log severity threshold [default: Info] -- `-l, --loose` Complete packing even if some files are missing [default: False] -- `-s, --source` Pack source files such as .map, editorimages, misc_models [default: False] -- `-sd, --shaderdebug` Print shader resolution details (Debug verbosity needed) [default: False] -- `-rd, --referencedebug` Print asset resolution details (Info verbosity needed) [default: False] -- `-f, --force` Overwrite existing files in the output path with impunity [default: False] -- `-i, --includepk3` Include pk3 files and pk3dirs in etmain when indexing files [default: False] -- `--ignore` Ignore some pk3 files or pk3dir directories [default: pak1.pk3|pak2.pk3|mp_bin.pk3] -- `-e, --exclude` Never pack files found in these pk3s or directories [default: pak0.pk3|pak0.pk3dir] -- `-m --mods` Adds pk3s in these mod folders to exclude-list -- `-?, -h, --help` Show help and usage information -- `--version` Show version information - -See below for examples. +#### `-o, --output` +Destination of the packing, defaults to `etmain`. Possible paths are filenames with `pk3` or `zip` extension, or directories. +File name in case of directory is `mapname.pk3`, or `.zip` if using `--source`. This setting is ignored if using `--dryrun`. + +#### `-d, --dryrun` +Run the packing operation without actually creating an archive. +Useful if you just want to discover what files would be packed or are missing, or want to see the size of the pk3. + +#### `-r --rename` +Name of the map after packing. Can be used to create different versions without changing project names, e.g. `mapname_b1`. +Among things renamed are BSP, mapscript, lightmap folder, levelshots files and shaders. + +#### `-v --verbosity` +The threshold for log messages to be printed. Default is `Info`, which may print too much or too little information depending +on your needs. Available options (from least to most verbose): `None`, `Fatal`, `Error`, `Warn`, `Info`, `Debug`, `Trace` + +#### `-l, --loose` +Creates the archive even if some files are missing. By default, missing files cause an error and don't result +in a created file. Use this setting with care if you know some files are fine to be missing. + +#### `-s, --source` +Pack a zip archive of source files instead of a map release, includes files such as .map, editorimages, misc_models, etc. + +#### `-f, --force` +Writes the output file even if it already exists. By default Pack3r doesn't overwrite existing pk3/zip files. + +#### `-m --mods` +Includes pk3s from mod folders when scanning for assets. Useful for things like tracemaps and speakerscripts that are +created in fs_game directory. + +#### `-sd, --shaderdebug` +Prints detailed information about which shaders are required by the map, and where they are referenced (at least `--verbosity Debug` needed) + +#### `-rd, --referencedebug` +Prints detailed information about which files are required by the map, and where they are referenced (at least `--verbosity Info` needed) + +#### `-p, --pk3, --includepk3` +Scan pk3 files and pk3dir-directories in etmain when indexing files (off by default for performance reasons). + +#### `-ns --noscan` +Don't scan these pk3s/directories at all when indexing files. + +#### `-np --nopack` +Scan these pk3s/directories, but don't pack their contents (for example pak0.pk3). + +#### `-?, -h, --help` +Prints help about usage and possible options, and their default values + +#### `--version` +Prints the build version, include this in bug reports + ## Limitations - Usable only through CLI, no GUI application is planned - Only brush primitives map format is supported (NetRadiant default) -- Shaders/textures are parsed from `ase`, `md3`, `mdc`, `skin` files. Other model formats such as `obj` are not yet supported (open an issue). +- Shaders/textures are coarsely parsed from `ase`, `md3`, `mdc`, `skin` files. Other model formats such as `obj` are not yet supported (open an issue). Models created by esoteric tools might not be parsed correctly even though radiant supports them - `terrain` shaders (1to2 etc) are not supported (open an issue) -- For performance reasons only a subset of files are considered for packing, see `PackableFile()` in file `Tokens.cs` +- For performance reasons only a subset of file extensions are packed: `tga|jpg|md3|mdc|mdm|ase|obj|fbx|shader|wav|roq|skin` ## File priority order -1. `pak0.pk3` (and other `--exclude` pk3s/directories), if a file or shader is found there, it won't be included in the release +1. `pak0.pk3` (and other `--nopack` pk3s/directories), if a file or shader is found there, it won't be included in the release 2. Files inside the _relative_ `etmain` of your map file (directory contaning `/maps`) 3. `etmain`, if the map is for example in `some.pk3dir/maps/mymap.map` 4. `pk3dir`-folders in `etmain`, in reverse alphabetical order @@ -62,12 +101,12 @@ Mapscript must be in `etmain/void.pk3dir/maps/` in this case and not directly in ## Example usage -### Pack a release-ready mymap.pk3 to etmain +### Pack a release-ready archive to `C:\ET\etmain\mymap.pk3` ```bash .\Pack3r 'C:\ET\etmain\maps\mymap.map' ``` -### Pack a release-ready beta release mymap_b1_.pk3 to etmain +### Pack a release-ready archive to `C:\ET\etmain\mymap_b1.pk3` with renamed bsp ```bash .\Pack3r 'C:\ET\etmain\maps\mymap.map' -r mymap_b1 .\Pack3r 'C:\ET\etmain\maps\mymap.map' --rename mymap_b1 @@ -86,7 +125,6 @@ Mapscript must be in `etmain/void.pk3dir/maps/` in this case and not directly in ``` ### Share map source with someone else -You can optionally delete bsp and lightmaps from the zip after packing to reduce file size ```bash .\Pack3r 'C:\ET\etmain\maps\mymap.map' -s -l -o 'C:\mymap_source.zip' .\Pack3r 'C:\ET\etmain\maps\mymap.map' --source --loose --output 'C:\mymap_source.zip' @@ -94,5 +132,5 @@ You can optionally delete bsp and lightmaps from the zip after packing to reduce ### Pack a map while ignoring some pk3s in etmain and including others ```bash -.\Pack3r 'C:\ET\etmain\maps\mymap.map' --includepk3 --ignore skies_MASTER.pk3 +.\Pack3r 'C:\ET\etmain\maps\mymap.map' --pk3 --noscan skies_MASTER.pk3 ```