From 0f520c5050dbf670e8ca5bfbfa4948deed7654da Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Thu, 5 Oct 2023 00:57:32 -0700 Subject: [PATCH 01/22] PPJ parsing --- CMakeLists.txt | 4 + Caprica/common/parser/CapricaPPJParser.cpp | 426 +++++++++++++++++++++ Caprica/common/parser/CapricaPPJParser.h | 21 + Caprica/common/parser/PapyrusProject.h | 286 ++++++++++++++ vcpkg.json | 3 +- 5 files changed, 739 insertions(+), 1 deletion(-) create mode 100644 Caprica/common/parser/CapricaPPJParser.cpp create mode 100644 Caprica/common/parser/CapricaPPJParser.h create mode 100644 Caprica/common/parser/PapyrusProject.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dc2fdf5..1c5a725 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,7 +97,9 @@ if(CAPRICA_STATIC_LIBRARY) add_library("${PROJECT_NAME}" STATIC ${HEADER_FILES} ${SOURCE_FILES}) find_package(Boost COMPONENTS container REQUIRED) find_package(fmt REQUIRED) + find_package(pugixml CONFIG REQUIRED) target_link_libraries(${PROJECT_NAME} PRIVATE Boost::container fmt::fmt) + target_link_libraries(${PROJECT_NAME} PRIVATE pugixml pugixml::static pugixml::pugixml) target_include_directories( "${PROJECT_NAME}" @@ -134,6 +136,7 @@ if(CAPRICA_STATIC_LIBRARY) else() find_package(Boost COMPONENTS filesystem program_options container REQUIRED) find_package(fmt REQUIRED) + find_package(pugixml CONFIG REQUIRED) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/Caprica ${Boost_INCLUDE_DIRS} @@ -141,6 +144,7 @@ else() add_subdirectory(Caprica) add_dependencies(${PROJECT_NAME} Caprica) target_link_libraries(${PROJECT_NAME} PRIVATE Boost::filesystem Boost::program_options Boost::container fmt::fmt) + target_link_libraries(${PROJECT_NAME} PRIVATE pugixml pugixml::static pugixml::pugixml) install( TARGETS Caprica diff --git a/Caprica/common/parser/CapricaPPJParser.cpp b/Caprica/common/parser/CapricaPPJParser.cpp new file mode 100644 index 0000000..a7cfab9 --- /dev/null +++ b/Caprica/common/parser/CapricaPPJParser.cpp @@ -0,0 +1,426 @@ +#include "CapricaPPJParser.h" +#include "common/CaselessStringComparer.h" +#include +#include +#include +#include +#include +#include +namespace caprica { +/** + * PapyrusProject.xsd: + ```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + */ + +/** + * Parses a PPJ file and returns a PapyrusProject object. + * The PPJ file is an XML file that describes a Papyrus project. + * See PapyrusProject.xsd for the schema. + */ + +static caseless_unordered_identifier_map GAME_MAP = { + {"sf1", PapyrusProject::PapyrusGame::Starfield }, + { "sse", PapyrusProject::PapyrusGame::SkyrimSpecialEdition}, + { "tes5", PapyrusProject::PapyrusGame::Skyrim }, + { "fo4", PapyrusProject::PapyrusGame::Fallout4 }, + { "f76", PapyrusProject::PapyrusGame::Fallout76 }, +}; +static caseless_unordered_identifier_map ASM_MAP = { + {"none", PapyrusProject::AsmType::None }, + { "keep", PapyrusProject::AsmType::Keep }, + { "only", PapyrusProject::AsmType::Only }, + { "discard", PapyrusProject::AsmType::Discard}, +}; + +static caseless_unordered_identifier_map BOOL_MAP = { + {"true", true }, + { "false", false}, + { "1", true }, + { "0", false}, +}; + +static std::vector rootAttributes { + "Game", "Output", "Flags", "Asm", "Optimize", "Release", "Final", "Anonymize", "Package", "Zip", + "xmlns" // ignore xmlns attribute +}; + +// variables are case insensitive and ascii-only +std::string CapricaPPJParser::substituteString(const char* text) { + if (strlen(text) == 0) + return text; + + std::string str = text; + if (str.find('@') == std::string::npos) + return str; + + std::string strLower = boost::to_lower_copy(str); + for (auto& varPair : variables) { + size_t pos = strLower.find(varPair.first); + while (pos != std::string::npos) { + str.replace(pos, varPair.first.size(), varPair.second); + strLower.replace(pos, varPair.first.size(), varPair.second); + pos = strLower.find(varPair.first, pos + 1); + } + } + return str; +} + +std::string CapricaPPJParser::ParseString(const pugi::xml_node& node) { + if (variables.size() == 0) + return node.text().as_string(); + return substituteString(node.text().as_string()); +} +std::string CapricaPPJParser::ParseString(const pugi::xml_attribute& attr) { + if (variables.size() == 0) + return attr.as_string(); + return substituteString(attr.as_string()); +} + +IncludeBase CapricaPPJParser::ParseIncludeBase(const pugi::xml_node& node) { + IncludeBase base; + base.name = ParseString(node.attribute("Name")); + base.rootDir = ParseString(node.attribute("RootDir")); + if (base.rootDir.empty()) + throw std::runtime_error("IncludeBase must have a RootDir attribute!"); + for (pugi::xml_node& child : node.children()) { + if (strcmp(child.name(), "Include") == 0) { + auto includePattern = Include { .path = ParseString(child) }; + includePattern.noRecurse = child.attribute("NoRecurse").as_bool(); + base.includes.emplace_back(includePattern); + } else if (strcmp(child.name(), "Match") == 0) { + auto match = Match { + .in = ParseString(child.attribute("In")), + .exclude = ParseString(child.attribute("Exclude")), + }; + match.path = ParseString(child.attribute("Path")); + match.noRecurse = child.attribute("NoRecurse").as_bool(); + base.matches.emplace_back(match); + } + } + return base; +} + +inline CommandList CapricaPPJParser::ParseCommandList(const pugi::xml_node& node) { + CommandList list; + list.description = ParseString(node.attribute("Description")); + list.useInBuild = node.attribute("UseInBuild").as_bool(); + for (pugi::xml_node& child : node.children()) + if (strcmp(child.name(), "Command") == 0) + list.commands.emplace_back(ParseString(child)); + return list; +} + +inline PapyrusProject::PapyrusGame ParseGameAttribute(const pugi::xml_node& root) { + std::string game = root.attribute("Game").as_string(); + if (game.empty()) + throw std::runtime_error("PapyrusProject must have a Game attribute!"); + auto it = GAME_MAP.find(game); + if (it == GAME_MAP.end()) + throw std::runtime_error("PapyrusProject has an invalid Game attribute!"); + return it->second; +} +inline PapyrusProject::AsmType ParseAsmAttribute(const pugi::xml_node& root) { + std::string asmAttr = root.attribute("Asm").as_string(); + if (asmAttr.empty()) + return PapyrusProject::AsmType::None; + auto it = ASM_MAP.find(asmAttr); + if (it == ASM_MAP.end()) + throw std::runtime_error("PapyrusProject has an invalid Asm attribute!"); + return it->second; +} +inline bool ParseBoolTypeAttribute(const pugi::xml_node& node, const char* name, bool defaultValue = false) { + std::string attr = node.attribute(name).as_string(); + if (attr.empty()) + return defaultValue; + auto it = BOOL_MAP.find(attr); + if (it == BOOL_MAP.end()) + throw std::runtime_error("PapyrusProject has an invalid " + std::string(name) + " attribute!"); + return it->second; +} + +bool CapricaPPJParser::ParseVariables(const pugi::xml_node& node) { + for (pugi::xml_node& child : node.children()) { + if (strcmp(child.name(), "Variable") == 0) { + std::string name = child.attribute("Name").as_string(); + boost::to_lower(name); + variables.emplace_back(std::pair { "@" + name, child.attribute("Value").as_string() }); + } + } + // now, sort the variables by length, so that we can replace the longest ones first + std::sort(variables.begin(), variables.end(), [](const auto& a, const auto& b) { + return a.first.size() > b.first.size(); + }); + return true; +} + +PapyrusProject CapricaPPJParser::Parse(const std::string& path) { + pugi::xml_document doc; + pugi::xml_parse_result result = doc.load_file(path.c_str()); + if (!result) { + std::cerr << "Error parsing PPJ file: " << result.description() << std::endl; + return {}; + } + pugi::xml_node root = doc.child("PapyrusProject"); + if (!root) { + std::cerr << "Error parsing PPJ file: No PapyrusProject node found!" << std::endl; + return {}; + } + PapyrusProject project; + // check if it has any other attributes besides the ones in the xsd + for (pugi::xml_attribute attr : root.attributes()) + if (std::find(rootAttributes.begin(), rootAttributes.end(), attr.name()) == rootAttributes.end()) + std::cerr << "Warning: PapyrusProject has an unknown attribute: " << attr.name() << std::endl; + + // convert to enum + project.game = ParseGameAttribute(root); + project.asmAttr = ParseAsmAttribute(root); + project.optimize = ParseBoolTypeAttribute(root, "Optimize"); + project.release = ParseBoolTypeAttribute(root, "Release"); + project.finalAttr = ParseBoolTypeAttribute(root, "Final"); + project.anonymize = ParseBoolTypeAttribute(root, "Anonymize"); + project.package = ParseBoolTypeAttribute(root, "Package"); + project.zip = ParseBoolTypeAttribute(root, "Zip"); + + // we have to do variables first + for (pugi::xml_node& child : root.children()) + if (strcmp(child.name(), "Variables") == 0) + ParseVariables(child); + + project.output = ParseString(root.attribute("Output")); + project.flags = ParseString(root.attribute("Flags")); + if (project.output.empty()) + throw std::runtime_error("PapyrusProject must have an Output attribute!"); + if (project.flags.empty()) + throw std::runtime_error("PapyrusProject must have a Flags attribute!"); + + for (pugi::xml_node& child : root.children()) { + if (strcmp(child.name(), "Imports") == 0) { + for (pugi::xml_node& import : child.children()) + if (strcmp(import.name(), "Import") == 0) + project.imports.emplace_back(); + } else if (strcmp(child.name(), "Folders") == 0) { + for (pugi::xml_node& folderElement : child.children()) { + if (strcmp(folderElement.name(), "Folder") == 0) { + auto flder = Folder { .path = ParseString(folderElement) }; + flder.noRecurse = folderElement.attribute("NoRecurse").as_bool(); + project.folders.emplace_back(flder); + } + } + } else if (strcmp(child.name(), "Scripts") == 0) { + for (pugi::xml_node& script : child.children()) + if (strcmp(script.name(), "Script") == 0) + project.scripts.emplace_back(ParseString(script)); + } // we do not care about the rest of these + // else if (strcmp(child.name(), "Packages") == 0) { + // project.packages.output = ParseString(child.attribute("Output")); + // for (pugi::xml_node& package : child.children()) { + // if (strcmp(package.name(), "Package") == 0) { + // Package pack = ParseIncludeBase(package); + // project.packages.packages.emplace_back(pack); + // } + // } + // } else if (strcmp(child.name(), "ZipFiles") == 0) { + // project.zipFiles.output = ParseString(child.attribute("Output")); + // for (pugi::xml_node& zipFileElement : child.children()) { + // if (strcmp(zipFileElement.name(), "ZipFile") == 0) { + // ZipFile zip = (ZipFile)ParseIncludeBase(zipFileElement); + // zip.compression = ParseString(zipFileElement.attribute("Compression")); + // project.zipFiles.zipFiles.emplace_back(zip); + // } + // } + // } else if (strcmp(child.name(), "PreBuildEvent") == 0) { + // project.preBuildEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostBuildEvent") == 0) { + // project.postBuildEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PreImportEvent") == 0) { + // project.preImportEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostImportEvent") == 0) { + // project.postImportEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PreCompileEvent") == 0) { + // project.preCompileEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostCompileEvent") == 0) { + // project.postCompileEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PreAnonymizeEvent") == 0) { + // project.preAnonymizeEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostAnonymizeEvent") == 0) { + // project.postAnonymizeEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PrePackageEvent") == 0) { + // project.prePackageEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostPackageEvent") == 0) { + // project.postPackageEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PreZipEvent") == 0) { + // project.preZipEvent = ParseCommandList(child); + // } else if (strcmp(child.name(), "PostZipEvent") == 0) { + // project.postZipEvent = ParseCommandList(child); + // } + } + return project; +} +} diff --git a/Caprica/common/parser/CapricaPPJParser.h b/Caprica/common/parser/CapricaPPJParser.h new file mode 100644 index 0000000..202a6d6 --- /dev/null +++ b/Caprica/common/parser/CapricaPPJParser.h @@ -0,0 +1,21 @@ +#pragma once +#include "pugixml.hpp" +#include +#include +#include +namespace caprica { + +class CapricaPPJParser { + + std::vector> variables; + std::string substituteString(const char* text); + std::string ParseString(const pugi::xml_node& node); + std::string ParseString(const pugi::xml_attribute& node); + IncludeBase ParseIncludeBase(const pugi::xml_node& node); + bool ParseVariables(const pugi::xml_node& node); + CommandList ParseCommandList(const pugi::xml_node& node); + +public: + PapyrusProject Parse(const std::string& path); +}; +} diff --git a/Caprica/common/parser/PapyrusProject.h b/Caprica/common/parser/PapyrusProject.h new file mode 100644 index 0000000..aa6fc0c --- /dev/null +++ b/Caprica/common/parser/PapyrusProject.h @@ -0,0 +1,286 @@ +#pragma once + +#include "common/GameID.h" + +#include +#include +#include +namespace caprica { + +/** + * PapyrusProject.xsd: + * + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + */ +/** + * Represents a Papyrus project (.ppj) file. + * + * See PapyrusProject.xsd for the schema. + */ +enum RecurseSetting { + NotSet = -1, + Recurse, + NoRecurse +}; +struct RecursablePath { + bool noRecurse = false; + std::string path = ""; +}; +using IncludePattern = RecursablePath; +struct MatchPattern : public RecursablePath { + std::string in = ""; + std::string exclude = ""; +}; +using Match = MatchPattern; +using Include = IncludePattern; + +struct IncludeBase { + std::string name; + std::string rootDir; + std::vector includes; + std::vector matches; +}; +struct IncludeZip : public IncludeBase { + std::string compression; +}; + +using Folder = RecursablePath; +using Package = IncludeBase; +using ZipFile = IncludeZip; + +using VariableList = std::map; +using ImportList = std::vector; +using FolderList = std::vector; +using ScriptList = std::vector; + +struct PackageList { + std::vector packages; + std::string output; +}; +struct ZipList { + std::vector zipFiles; + std::string output; +}; +struct CommandList { + std::vector commands; + std::string description; + bool useInBuild = false; +}; + +struct PapyrusProject { + // different values than GameID + enum class PapyrusGame : uint16_t { + INVALID = 0xFFFF, + UNKNOWN = 0, + Skyrim = 1, + Fallout4 = 2, + Fallout76 = 3, + Starfield = 4, + SkyrimSpecialEdition = 5 + }; + + enum class AsmType : uint8_t { + None, + Keep, + Only, + Discard + }; + + // Attributes + PapyrusGame game = PapyrusGame::UNKNOWN; + bool optimize = false; + bool release = false; + bool finalAttr = false; + bool anonymize = false; + bool package = false; + bool zip = false; + std::string output; // required + std::string flags; // required + AsmType asmAttr = AsmType::None; + + // elements + VariableList variables; + ImportList imports; + FolderList folders; + ScriptList scripts; + PackageList packages; + ZipList zipFiles; + CommandList preBuildEvent; + CommandList postBuildEvent; + CommandList preImportEvent; + CommandList postImportEvent; + CommandList preCompileEvent; + CommandList postCompileEvent; + CommandList preAnonymizeEvent; + CommandList postAnonymizeEvent; + CommandList prePackageEvent; + CommandList postPackageEvent; + CommandList preZipEvent; + CommandList postZipEvent; +}; +} // namespace caprica diff --git a/vcpkg.json b/vcpkg.json index 28d0e42..671903b 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -17,7 +17,8 @@ "boost-filesystem", "boost-program-options", "boost-property-tree", - "fmt" + "fmt", + "pugixml" ] } }, From 07e487689572e2ee07cd12e9685f2f743336d027 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Thu, 5 Oct 2023 00:57:54 -0700 Subject: [PATCH 02/22] initial loading of PPJ --- Caprica/common/CapricaConfig.h | 9 ++- Caprica/main_options.cpp | 116 +++++++++++++++++++++++++-------- 2 files changed, 96 insertions(+), 29 deletions(-) diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index 22d08ad..c3654eb 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -5,8 +5,9 @@ #include #include -#include #include "GameID.h" +#include +#include namespace caprica { namespace conf { @@ -17,7 +18,11 @@ namespace General { extern bool compileInParallel; // If true, only report failures, not progress. extern bool quietCompile; -} + // self-explanatory + extern std::string outputDirectory; + // If true, remove identifying information from the header. + extern bool anonymizeOutput; + } // options related to compatibility with PCompiler's CLI parsing and name resolution namespace PCompiler { diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index c026614..81b8257 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -13,7 +14,6 @@ #include #include #include - namespace conf = caprica::conf; namespace po = boost::program_options; namespace filesystem = std::filesystem; @@ -103,6 +103,24 @@ void SaveDefaultConfig(const po::options_description& descOptions, // or write_ini boost::property_tree::write_ini(configFile, tree); } + +std::string +findFlags(const std::string& flagsPath, const std::string& progamBasePath, const std::string& baseOutputDir) { + if (filesystem::exists(flagsPath)) + return flagsPath; + + for (auto& i : conf::Papyrus::importDirectories) + if (filesystem::exists(i + "\\" + flagsPath)) + return i + "\\" + flagsPath; + + if (filesystem::exists(baseOutputDir + "\\" + flagsPath)) + return baseOutputDir + "\\" + flagsPath; + if (filesystem::exists(progamBasePath + "\\" + flagsPath)) + return progamBasePath + "\\" + flagsPath; + + return ""; +}; + bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManager* jobManager) { try { bool iterateCompiledDirectoriesRecursively = false; @@ -322,6 +340,72 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << visibleDesc << std::endl; return false; } + // ensure cwd is placed first if this isn't turned on; this isn't overridden by the ppj + if (!vm["ignorecwd"].as()) { + conf::Papyrus::importDirectories.reserve(1); + conf::Papyrus::importDirectories.emplace_back(filesystem::current_path().string()); + if (!conf::General::quietCompile) { + std::cout << "Adding current working directory to import list: " << conf::Papyrus::importDirectories[0] + << std::endl; + } + } + + // we have to put this at the beginning, because flags passed override this + PapyrusProject ppj; + auto inputFiles = vm["input-file"].as>(); + for (auto& f : inputFiles) { + auto ext = filesystem::path(f).extension().string(); + if (ext == ".ppj") { + if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + std::cout << "Only one project file can be specified!" << std::endl; + return false; + } + CapricaPPJParser ppjParser; + ppj = ppjParser.Parse(f); + std::cout << "Loaded project." << std::endl; + for (auto& f : ppj.folders) + std::cout << "Folder: " << f.path << std::endl; + + for (auto& s : ppj.scripts) + std::cout << "Script: " << s << std::endl; + } + } + + if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + // set all the options in the ppj + conf::Papyrus::game = + ppj.game == PapyrusProject::PapyrusGame::SkyrimSpecialEdition ? GameID::Skyrim : (GameID)ppj.game; + conf::General::anonymizeOutput = ppj.anonymize; + conf::General::outputDirectory = ppj.output; + if (conf::Papyrus::importDirectories.empty()) { + conf::Papyrus::importDirectories = ppj.imports; + } else { + conf::Papyrus::importDirectories.insert(conf::Papyrus::importDirectories.end(), + ppj.imports.begin(), + ppj.imports.end()); + } + conf::CodeGeneration::disableDebugCode = ppj.release; + conf::CodeGeneration::disableBetaCode = ppj.finalAttr; + conf::CodeGeneration::enableOptimizations = ppj.optimize; + + switch (ppj.asmAttr) { + case PapyrusProject::AsmType::None: + case PapyrusProject::AsmType::Discard: + conf::Debug::dumpPexAsm = false; + break; + case PapyrusProject::AsmType::Keep: + case PapyrusProject::AsmType::Only: // TODO: HANDLE THIS + conf::Debug::dumpPexAsm = true; + break; + } + for (auto& f : ppj.folders) + inputFiles.push_back(f.path); + for (auto& f : ppj.scripts) + inputFiles.push_back(f); + + // handle the inputs + } + // set the options according to this std::string gameType = vm["game"].as(); if (_stricmp(gameType.c_str(), "Starfield") == 0) { @@ -415,14 +499,6 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage conf::Debug::dumpPexAsm = false; } - if (!vm["ignorecwd"].as()) { - conf::Papyrus::importDirectories.reserve(1); - conf::Papyrus::importDirectories.emplace_back(filesystem::current_path().string()); - if (!conf::General::quietCompile) { - std::cout << "Adding current working directory to import list: " << conf::Papyrus::importDirectories[0] - << std::endl; - } - } if (vm.count("import")) { auto dirs = vm["import"].as>(); @@ -459,25 +535,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage baseOutputDir = FSUtils::canonical(baseOutputDir); if (vm.count("flags")) { - const auto findFlags = [progamBasePath, baseOutputDir](const std::string& flagsPath) -> std::string { - if (filesystem::exists(flagsPath)) - return flagsPath; - - for (auto& i : conf::Papyrus::importDirectories) - if (filesystem::exists(i + "\\" + flagsPath)) - return i + "\\" + flagsPath; - - if (filesystem::exists(baseOutputDir + "\\" + flagsPath)) - return baseOutputDir + "\\" + flagsPath; - if (filesystem::exists(progamBasePath + "\\" + flagsPath)) - return progamBasePath + "\\" + flagsPath; - - return ""; - }; + auto flags = vm["flags"].as(); - auto flagsPath = findFlags(vm["flags"].as()); + auto flagsPath = findFlags(flags, progamBasePath, baseOutputDir); if (flagsPath == "") { - std::cout << "Unable to locate flags file '" << vm["flags"].as() << "'." << std::endl; + std::cout << "Unable to locate flags file '" << flags << "'." << std::endl; return false; } @@ -546,7 +608,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } } else { std::string_view ext = FSUtils::extensionAsRef(f); - if (!pathEq(ext, ".psc") && !pathEq(ext, ".pas") && !pathEq(ext, ".pex")) { + if (!pathEq(ext, ".psc") && !pathEq(ext, ".pas") && !pathEq(ext, ".pex") && !pathEq(ext, ".ppj")) { std::cout << "Don't know how to handle input file '" << f << "'!" << std::endl; std::cout << "Expected either a Papyrus file (*.psc), Pex assembly file (*.pas), or a Pex file (*.pex)!" << std::endl; From 19cac6181a25aa0fb582c4b93764e1ece0e0bb25 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:37:10 -0700 Subject: [PATCH 03/22] fix output option `-o` was going to `--optimize` instead of `--output` --- Caprica/main_options.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index 81b8257..4bed94f 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -135,10 +135,10 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage "Set the compiler's import directories.") ("flags,f", po::value(), "Set the file defining the user flags.") + ("output,o", po::value(), + "Set the directory to save compiler output to.") ("optimize,op,O", po::bool_switch(&conf::CodeGeneration::enableOptimizations)->default_value(false), "Enable optimizations.") - ("output,o", po::value()->default_value(filesystem::current_path().string()), - "Set the directory to save compiler output to.") ("parallel-compile,p", po::bool_switch(&conf::General::compileInParallel)->default_value(false), "Compile files in parallel.") ("recurse,R", po::bool_switch(&iterateCompiledDirectoriesRecursively)->default_value(false), From 6ca054d393bb62088b5719c899c6b25b7c7a1758 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:12:44 -0700 Subject: [PATCH 04/22] ignore whitespace on detecting script name --- Caprica/papyrus/PapyrusCompilationContext.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Caprica/papyrus/PapyrusCompilationContext.cpp b/Caprica/papyrus/PapyrusCompilationContext.cpp index aed6437..dccfea1 100644 --- a/Caprica/papyrus/PapyrusCompilationContext.cpp +++ b/Caprica/papyrus/PapyrusCompilationContext.cpp @@ -119,15 +119,14 @@ void PapyrusCompilationNode::FileReadJob::run() { } } -std::string_view findScriptName(const std::string_view& data, const std::string_view& startstring, bool stripWhitespace = false) -{ +std::string_view findScriptName(const std::string_view& data, const std::string_view& startstring) { size_t last = 0; while (last < data.size()) { auto next = data.find('\n', last); if (next == std::string_view::npos) next = data.size() - 1; auto line = data.substr(last, next - last); - auto begin = stripWhitespace ? line.find_first_not_of(" \t") : 0; + auto begin = line.find_first_not_of(" \t"); if (strnicmp(line.substr(begin, startstring.size()).data(), startstring.data(), startstring.size()) == 0) { auto first = line.find_first_not_of(" \t", startstring.size()); return line.substr(first, line.find_first_of(" \t", first) - first); @@ -152,7 +151,7 @@ void PapyrusCompilationNode::FilePreParseJob::run() { CapricaReportingContext::logicalFatal("Unable to find script name in '{}'.", parent->sourceFilePath); parent->objectName = parent->pexFile->getStringValue(parent->pexFile->objects.front()->name).to_string(); } else if (pathEq(ext, ".pas")) { - parent->objectName = findScriptName(parent->readFileData, ".object", true); + parent->objectName = findScriptName(parent->readFileData, ".object"); } else { CapricaReportingContext::logicalFatal("Unable to determine the type of file to load '{}' as.", parent->reportedName); From 00b70e425a7ddac10764260f9c8d37396caaba1f Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:16:23 -0700 Subject: [PATCH 05/22] finish PPJ support --- Caprica/common/CapricaConfig.cpp | 73 ++++- Caprica/common/CapricaConfig.h | 40 ++- Caprica/common/parser/CapricaPPJParser.cpp | 21 +- Caprica/common/parser/PapyrusProject.h | 29 +- Caprica/main.cpp | 80 ++---- Caprica/main_options.cpp | 295 +++++++++++++-------- 6 files changed, 340 insertions(+), 198 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index c5bb91e..97e2507 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -1,5 +1,7 @@ +#include "CapricaConfig.h" #include - +#include +#include namespace caprica { namespace conf { // These should always be defaulted to false/empty, and their real @@ -8,10 +10,14 @@ namespace caprica { namespace conf { namespace General { bool compileInParallel{ false }; bool quietCompile{ false }; -} + bool recursive { false }; + std::string outputDirectory; + bool anonymizeOutput; + std::vector inputFiles; + } -namespace PCompiler { - // pCompiler compatibility mode. + namespace PCompiler { + // pCompiler compatibility mode. bool pCompilerCompatibilityMode{false}; bool all{false}; bool norecurse{false}; @@ -46,9 +52,9 @@ namespace EngineLimits { } namespace Papyrus { - GameID game; - bool allowCompilerIdentifiers{ false }; - bool allowDecompiledStructNameRefs{ false }; +GameID game { GameID::UNKNOWN }; +bool allowCompilerIdentifiers { false }; +bool allowDecompiledStructNameRefs{ false }; bool allowNegativeLiteralAsBinaryOp{ false }; bool enableLanguageExtensions{ false }; bool ignorePropertyNameLocalConflicts{ false }; @@ -81,4 +87,55 @@ namespace Warnings { std::unordered_set warningsToEnable{ }; } -}} + std::filesystem::path InputFile::resolved_relative() const { + for (auto& dir : Papyrus::importDirectories) + if (dirContains(dir)) + return get_relative_path(dir); + return {}; // not found + } + + std::filesystem::path InputFile::resolved_absolute() const { + // find the file among the import directories + for (auto& dir : Papyrus::importDirectories) + if (dirContains(dir)) + return get_absolute_path(dir); + return {}; // not found + } + + std::filesystem::path InputFile::resolved_absolute_basedir() const { + // find the file among the import directories + for (auto& dir : Papyrus::importDirectories) + if (dirContains(dir)) + return dir; + return {}; // not found + } + + std::filesystem::path InputFile::get_absolute_path(const std::filesystem::path& absDir) const { + if (path.is_absolute()) + return FSUtils::canonical(path.string()); + else + return FSUtils::canonical((absDir / path).string()); + } + + std::filesystem::path InputFile::get_relative_path(const std::filesystem::path& absDir) const { + std::filesystem::path cpath; + if (path.is_absolute()) + cpath = FSUtils::canonical(path.string()); + else + cpath = FSUtils::canonical((absDir / path).string()); + return cpath.lexically_relative(absDir); + } + + bool InputFile::dirContains(const std::filesystem::path& dir) const { + + if (path.is_absolute()) { + // check if the path is contained in the import directory + if (!path.lexically_relative(dir).string().starts_with("..")) + return true; + } else { + if (std::filesystem::exists(dir / path)) + return true; + } + return false; + } + }} diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index c3654eb..c48a2a7 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -8,9 +8,41 @@ #include "GameID.h" #include #include - +#include +#include namespace caprica { namespace conf { +struct InputFile { + bool noRecurse = true; + std::filesystem::path path; + std::filesystem::path cwd; + bool resolved = false; + std::filesystem::path resolved_relative() const; + std::filesystem::path resolved_absolute() const; + std::filesystem::path resolved_absolute_basedir() const; + InputFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = std::filesystem::current_path()) + : noRecurse(noRecurse), + path(std::move(_path)), + cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { + + //special handler; this points to something relative to the cwd, not an object path to be resolved + if (path.string().starts_with(".\\") || path.string().starts_with("./") || path.string().contains("..")) + { + if (!path.is_absolute()){ + path = cwd / path; + } + path = FSUtils::canonical(path.string()); + } + } + + private: + std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; + std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; + bool dirContains(const std::filesystem::path& dir) const; +}; + // Options that don't fit in any other category. namespace General { // If true, when compiling multiple files, do so @@ -18,11 +50,15 @@ namespace General { extern bool compileInParallel; // If true, only report failures, not progress. extern bool quietCompile; + // If true, recurse into subdirectories when compiling. + extern bool recursive; // self-explanatory extern std::string outputDirectory; // If true, remove identifying information from the header. extern bool anonymizeOutput; - } + // input files + extern std::vector inputFiles; +} // options related to compatibility with PCompiler's CLI parsing and name resolution namespace PCompiler { diff --git a/Caprica/common/parser/CapricaPPJParser.cpp b/Caprica/common/parser/CapricaPPJParser.cpp index a7cfab9..ead3d43 100644 --- a/Caprica/common/parser/CapricaPPJParser.cpp +++ b/Caprica/common/parser/CapricaPPJParser.cpp @@ -211,6 +211,16 @@ static std::vector rootAttributes { "xmlns" // ignore xmlns attribute }; +inline BoolSetting ParseBoolTypeAttribute(const pugi::xml_node& node, const char* name) { + std::string attr = node.attribute(name).as_string(); + if (attr.empty()) + return BoolSetting::NOT_SET; + auto it = BOOL_MAP.find(attr); + if (it == BOOL_MAP.end()) + throw std::runtime_error("PapyrusProject has an invalid " + std::string(name) + " attribute!"); + return it->second ? BoolSetting::True : BoolSetting::False; +} + // variables are case insensitive and ascii-only std::string CapricaPPJParser::substituteString(const char* text) { if (strlen(text) == 0) @@ -295,15 +305,6 @@ inline PapyrusProject::AsmType ParseAsmAttribute(const pugi::xml_node& root) { throw std::runtime_error("PapyrusProject has an invalid Asm attribute!"); return it->second; } -inline bool ParseBoolTypeAttribute(const pugi::xml_node& node, const char* name, bool defaultValue = false) { - std::string attr = node.attribute(name).as_string(); - if (attr.empty()) - return defaultValue; - auto it = BOOL_MAP.find(attr); - if (it == BOOL_MAP.end()) - throw std::runtime_error("PapyrusProject has an invalid " + std::string(name) + " attribute!"); - return it->second; -} bool CapricaPPJParser::ParseVariables(const pugi::xml_node& node) { for (pugi::xml_node& child : node.children()) { @@ -364,7 +365,7 @@ PapyrusProject CapricaPPJParser::Parse(const std::string& path) { if (strcmp(child.name(), "Imports") == 0) { for (pugi::xml_node& import : child.children()) if (strcmp(import.name(), "Import") == 0) - project.imports.emplace_back(); + project.imports.emplace_back(ParseString(import)); } else if (strcmp(child.name(), "Folders") == 0) { for (pugi::xml_node& folderElement : child.children()) { if (strcmp(folderElement.name(), "Folder") == 0) { diff --git a/Caprica/common/parser/PapyrusProject.h b/Caprica/common/parser/PapyrusProject.h index aa6fc0c..ed3d220 100644 --- a/Caprica/common/parser/PapyrusProject.h +++ b/Caprica/common/parser/PapyrusProject.h @@ -187,6 +187,13 @@ enum RecurseSetting { Recurse, NoRecurse }; + +enum class BoolSetting : uint8_t { + False, + True, + NOT_SET = 0xFF, +}; + struct RecursablePath { bool noRecurse = false; std::string path = ""; @@ -241,27 +248,29 @@ struct PapyrusProject { Fallout4 = 2, Fallout76 = 3, Starfield = 4, - SkyrimSpecialEdition = 5 + SkyrimSpecialEdition = 0xFF01, + NOT_SET = UNKNOWN }; enum class AsmType : uint8_t { None, Keep, Only, - Discard + Discard, + NOT_SET = 0xFF, }; // Attributes - PapyrusGame game = PapyrusGame::UNKNOWN; - bool optimize = false; - bool release = false; - bool finalAttr = false; - bool anonymize = false; - bool package = false; - bool zip = false; + PapyrusGame game = PapyrusGame::NOT_SET; + BoolSetting optimize = BoolSetting::NOT_SET; + BoolSetting release = BoolSetting::NOT_SET; + BoolSetting finalAttr = BoolSetting::NOT_SET; + BoolSetting anonymize = BoolSetting::NOT_SET; + BoolSetting package = BoolSetting::NOT_SET; + BoolSetting zip = BoolSetting::NOT_SET; std::string output; // required std::string flags; // required - AsmType asmAttr = AsmType::None; + AsmType asmAttr = AsmType::NOT_SET; // elements VariableList variables; diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 3dc4594..ea0619a 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -107,17 +107,18 @@ PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType time_t lastModTime, size_t fileSize); -bool addSingleFile(const std::string &f, +bool addSingleFile(const conf::InputFile& input, const std::string &baseOutputDir, caprica::CapricaJobManager *jobManager, PapyrusCompilationNode::NodeType nodeType); -bool addFilesFromDirectory(const std::string &f, +bool addFilesFromDirectory(const std::string& f, bool recursive, - const std::string &baseOutputDir, - caprica::CapricaJobManager *jobManager, + const std::string& baseOutputDir, + caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType, - const std::string &startingNS = "") { + const std::string& subdir = "", + const std::string& startingNS = "") { // Blargle flargle.... Using the raw Windows API is 5x // faster than boost::filesystem::recursive_directory_iterator, // at 40ms vs. 200ms for the boost solution, and the raw API @@ -125,7 +126,7 @@ bool addFilesFromDirectory(const std::string &f, // last write time, and file size, all without any extra processing. auto absBaseDir = caprica::FSUtils::canonical(f); std::vector dirs{}; - dirs.push_back("\\"); + dirs.push_back(subdir.empty() ? "\\" : "\\" + subdir); auto baseDirMap = getBaseSigMap(conf::Papyrus::game); auto l_startNS = startingNS; while (dirs.size()) { @@ -213,7 +214,8 @@ bool addFilesFromDirectory(const std::string &f, // if it's true, then this is the base dir and the namespace should be root auto namespaceName = l_startNS + curDir; std::replace(namespaceName.begin(), namespaceName.end(), '\\', ':'); - namespaceName = namespaceName.substr(1); + while (namespaceName[0] == ':') + namespaceName = namespaceName.substr(1); caprica::papyrus::PapyrusCompilationContext::pushNamespaceFullContents(namespaceName, std::move(namespaceMap)); } } else { @@ -315,7 +317,7 @@ bool handleImports(const std::vector &f, caprica::CapricaJobManager std::string ns = ""; if (conf::Papyrus::game > GameID::Skyrim) ns = "!!temp" + std::to_string(i++); - if (!addFilesFromDirectory(dir, true, "", jobManager, PapyrusCompilationNode::NodeType::PapyrusImport, ns)) + if (!addFilesFromDirectory(dir, true, "", jobManager, PapyrusCompilationNode::NodeType::PapyrusImport, "", ns)) return false; abs_import_dirs.emplace(std::filesystem::absolute(dir).string()); } @@ -324,61 +326,37 @@ bool handleImports(const std::vector &f, caprica::CapricaJobManager return true; } -bool addSingleFile(const std::string &f, - const std::string &baseOutputDir, - caprica::CapricaJobManager *jobManager, +bool addSingleFile(const conf::InputFile& input, + const std::string& baseOutputDir, + caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType) { // Get the file size and last modified time using std::filesystem std::error_code ec; - if (!std::filesystem::exists(f)) { - std::cout << "ERROR: File '" << f << "' does not exist!" << std::endl; + + auto relPath = input.resolved_relative(); + std::string namespaceDir = relPath.parent_path().string(); + auto absPath = input.resolved_absolute(); + auto filename = absPath.filename().string(); + std::string absBaseDir = input.resolved_absolute_basedir().string() + "\\"; + if (!std::filesystem::exists(absPath)) { + std::cout << "ERROR: Resolved file path '" << absPath + << "' is not in an import directory, cannot resolve namespace!" << std::endl; return false; } - auto lastModTime = std::filesystem::last_write_time(f, ec); + auto lastModTime = std::filesystem::last_write_time(absPath, ec); if (ec) { - std::cout << "An error occurred while trying to get the last modified time of '" << f << "'!" << std::endl; + std::cout << "An error occurred while trying to get the last modified time of '" << absPath << "'!" << std::endl; return false; } - auto fileSize = std::filesystem::file_size(f, ec); + auto fileSize = std::filesystem::file_size(absPath, ec); if (ec) { - std::cout << "An error occurred while trying to get the file size of '" << f << "'!" << std::endl; + std::cout << "An error occurred while trying to get the file size of '" << absPath << "'!" << std::endl; return false; } - - std::string namespaceDir = "\\"; - auto path = std::filesystem::path(f); - auto filename = std::string(caprica::FSUtils::filenameAsRef(f)); - std::string absBaseDir = std::filesystem::absolute(f).parent_path().string(); - if (!path.is_absolute()) { - auto ppath = caprica::FSUtils::parentPathAsRef(f); - if (ppath.compare(filename.c_str()) != 0) { - namespaceDir += ppath; - absBaseDir = absBaseDir.rfind(namespaceDir) != std::string::npos - ? absBaseDir.substr(0, absBaseDir.rfind(namespaceDir)) - : absBaseDir; - } - } else { - // PCompiler-like namespace resolution by scanning imports - bool found = false; - for (auto& dir : abs_import_dirs) { - auto relPath = std::filesystem::relative(f, dir); - // check if relpath begins with ".." - if (relPath != path && !relPath.string().starts_with("..")) { - namespaceDir += relPath.parent_path().string(); - absBaseDir = dir; - found = true; - break; - } - } - if (!found) { - std::cout << "ERROR: Absolute file path '" << f << "' is not in an import directory, cannot resolve namespace!" - << std::endl; - return false; - } - } - auto namespaceName = namespaceDir; + auto namespaceName = relPath.parent_path().string(); std::replace(namespaceName.begin(), namespaceName.end(), '\\', ':'); - namespaceName = namespaceName.substr(1); + if (namespaceName[0] == ':') + namespaceName = namespaceName.substr(1); if (!conf::General::quietCompile) std::cout << "Adding file '" << filename << "' to namespace '" << namespaceName << "'." << std::endl; auto node = getNode(nodeType, diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index 4bed94f..ab13210 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -25,11 +25,12 @@ bool addFilesFromDirectory(const std::string& f, bool recursive, const std::string& baseOutputDir, caprica::CapricaJobManager* jobManager, - caprica::papyrus::PapyrusCompilationNode::NodeType nodeType, + papyrus::PapyrusCompilationNode::NodeType nodeType, + const std::string& subdir = "", const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); -bool addSingleFile(const std::string& f, +bool addSingleFile(const conf::InputFile& input, const std::string& baseOutputDir, caprica::CapricaJobManager* jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType nodeType); @@ -121,6 +122,11 @@ findFlags(const std::string& flagsPath, const std::string& progamBasePath, const return ""; }; +void setPPJBool(BoolSetting value, bool& setting) { + if (value != BoolSetting::NOT_SET) + setting = value == BoolSetting::True ? true : false; +}; + bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManager* jobManager) { try { bool iterateCompiledDirectoriesRecursively = false; @@ -129,7 +135,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage po::options_description desc("General"); desc.add_options() ("help,h,?", "Print usage information.") - ("game,g", po::value()->default_value("starfield"), + ("game,g", po::value(), "Set the game type to compile for. Valid values are: starfield, skyrim, fallout4, fallout76. (default: starfield)") ("import,i", po::value>()->composing(), "Set the compiler's import directories.") @@ -141,10 +147,10 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage "Enable optimizations.") ("parallel-compile,p", po::bool_switch(&conf::General::compileInParallel)->default_value(false), "Compile files in parallel.") - ("recurse,R", po::bool_switch(&iterateCompiledDirectoriesRecursively)->default_value(false), - "Recursively compile all scripts in the directories passed.") ("release,r", po::bool_switch(&conf::CodeGeneration::disableDebugCode)->default_value(false), "Don't generate DebugOnly code.") + ("recurse,R", po::bool_switch(&iterateCompiledDirectoriesRecursively)->default_value(false), + "Recursively compile all scripts in the directories passed.") ("final", po::bool_switch(&conf::CodeGeneration::disableBetaCode)->default_value(false), "Don't generate BetaOnly code.") ("all-warnings-as-errors", @@ -340,54 +346,76 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << visibleDesc << std::endl; return false; } - // ensure cwd is placed first if this isn't turned on; this isn't overridden by the ppj - if (!vm["ignorecwd"].as()) { - conf::Papyrus::importDirectories.reserve(1); - conf::Papyrus::importDirectories.emplace_back(filesystem::current_path().string()); - if (!conf::General::quietCompile) { - std::cout << "Adding current working directory to import list: " << conf::Papyrus::importDirectories[0] - << std::endl; + + auto filesPassed = std::vector(); + for (auto& f : vm["input-file"].as>()) { + if (f.find(';') != std::string::npos) { + std::istringstream s(f); + std::string sd; + while (getline(s, sd, ';')) + filesPassed.push_back(sd); + } else { + filesPassed.push_back(f); } } + std::string flags; + std::string baseOutputDir; // we have to put this at the beginning, because flags passed override this PapyrusProject ppj; - auto inputFiles = vm["input-file"].as>(); - for (auto& f : inputFiles) { + std::string ppjPath; + auto filesToRemove = std::vector(); + for (auto& f : filesPassed) { auto ext = filesystem::path(f).extension().string(); if (ext == ".ppj") { - if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + if (!ppjPath.empty()) { std::cout << "Only one project file can be specified!" << std::endl; return false; } CapricaPPJParser ppjParser; ppj = ppjParser.Parse(f); - std::cout << "Loaded project." << std::endl; - for (auto& f : ppj.folders) - std::cout << "Folder: " << f.path << std::endl; - - for (auto& s : ppj.scripts) - std::cout << "Script: " << s << std::endl; + filesToRemove.push_back(f); + ppjPath = f; } } + // Remove the project file from the input files. + for (auto& f : filesToRemove) { + auto it = std::find(filesPassed.begin(), filesPassed.end(), f); + if (it != filesPassed.end()) + filesPassed.erase(it); + } - if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + if (vm.count("pcompiler")) { + conf::PCompiler::pCompilerCompatibilityMode = vm["pcompiler"].as(); + conf::PCompiler::all = vm["all"].as(); + conf::PCompiler::norecurse = vm["norecurse"].as(); + if (conf::PCompiler::norecurse) + iterateCompiledDirectoriesRecursively = false; + } + + if (!ppjPath.empty()) { + auto baseDir = filesystem::path(ppjPath).parent_path(); + if (!baseDir.empty()) + baseDir = FSUtils::canonical(baseDir.string()); + else + baseDir = filesystem::current_path(); // set all the options in the ppj - conf::Papyrus::game = - ppj.game == PapyrusProject::PapyrusGame::SkyrimSpecialEdition ? GameID::Skyrim : (GameID)ppj.game; - conf::General::anonymizeOutput = ppj.anonymize; - conf::General::outputDirectory = ppj.output; - if (conf::Papyrus::importDirectories.empty()) { - conf::Papyrus::importDirectories = ppj.imports; - } else { - conf::Papyrus::importDirectories.insert(conf::Papyrus::importDirectories.end(), - ppj.imports.begin(), - ppj.imports.end()); + if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + conf::Papyrus::game = + ppj.game == PapyrusProject::PapyrusGame::SkyrimSpecialEdition ? GameID::Skyrim : (GameID)ppj.game; } - conf::CodeGeneration::disableDebugCode = ppj.release; - conf::CodeGeneration::disableBetaCode = ppj.finalAttr; - conf::CodeGeneration::enableOptimizations = ppj.optimize; + conf::Papyrus::importDirectories.reserve(conf::Papyrus::importDirectories.size() + ppj.imports.size()); + for (auto& i : ppj.imports) { + std::filesystem::path ipath = i; + if (!ipath.is_absolute()) + ipath = (baseDir / i); + conf::Papyrus::importDirectories.emplace_back(FSUtils::canonical(ipath.string())); + } + setPPJBool(ppj.anonymize, conf::General::anonymizeOutput); + setPPJBool(ppj.release, conf::CodeGeneration::disableDebugCode); + setPPJBool(ppj.finalAttr, conf::CodeGeneration::disableBetaCode); + setPPJBool(ppj.optimize, conf::CodeGeneration::enableOptimizations); switch (ppj.asmAttr) { case PapyrusProject::AsmType::None: case PapyrusProject::AsmType::Discard: @@ -397,28 +425,60 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage case PapyrusProject::AsmType::Only: // TODO: HANDLE THIS conf::Debug::dumpPexAsm = true; break; + default: + break; } + auto outputDir = std::filesystem::path(ppj.output); + if (outputDir.is_relative()) + outputDir = (baseDir / ppj.output); + baseOutputDir = outputDir.string(); + + conf::General::inputFiles.reserve(conf::General::inputFiles.size() + ppj.folders.size() + ppj.scripts.size()); for (auto& f : ppj.folders) - inputFiles.push_back(f.path); + conf::General::inputFiles.emplace_back(f.path, f.noRecurse, baseDir); for (auto& f : ppj.scripts) - inputFiles.push_back(f); - - // handle the inputs + conf::General::inputFiles.emplace_back(f, false, baseDir); + flags = ppj.flags; + } else { // if we're not doing a project file... + // ensure cwd is placed first + if (!vm["ignorecwd"].as()) { + conf::Papyrus::importDirectories.reserve(1); + conf::Papyrus::importDirectories.emplace_back(filesystem::current_path().string()); + if (!conf::General::quietCompile) { + std::cout << "Adding current working directory to import list: " << conf::Papyrus::importDirectories[0] + << std::endl; + } + } } - // set the options according to this - - std::string gameType = vm["game"].as(); - if (_stricmp(gameType.c_str(), "Starfield") == 0) { - conf::Papyrus::game = GameID::Starfield; - } else if (_stricmp(gameType.c_str(), "Skyrim") == 0) { - conf::Papyrus::game = GameID::Skyrim; - } else if (_stricmp(gameType.c_str(), "Fallout4") == 0) { - conf::Papyrus::game = GameID::Fallout4; - } else if (_stricmp(gameType.c_str(), "Fallout76") == 0) { - conf::Papyrus::game = GameID::Fallout76; - } else { - std::cout << "Unrecognized game type '" << gameType << "'!" << std::endl; - return false; + if (conf::Papyrus::game == GameID::UNKNOWN) { + std::string gameType; + if (vm.count("game")) + gameType = vm["game"].as(); + else + gameType = "Starfield"; + if (_stricmp(gameType.c_str(), "Starfield") == 0) { + conf::Papyrus::game = GameID::Starfield; + } else if (_stricmp(gameType.c_str(), "Skyrim") == 0) { + conf::Papyrus::game = GameID::Skyrim; + } else if (_stricmp(gameType.c_str(), "Fallout4") == 0) { + conf::Papyrus::game = GameID::Fallout4; + } else if (_stricmp(gameType.c_str(), "Fallout76") == 0) { + conf::Papyrus::game = GameID::Fallout76; + } else { + std::cout << "Unrecognized game type '" << gameType << "'!" << std::endl; + return false; + } + } + + if (vm.count("flags")) + flags = vm["flags"].as(); + if (vm.count("output")) + baseOutputDir = vm["output"].as(); + else if (baseOutputDir.empty()) + baseOutputDir = filesystem::current_path().string(); + if (vm.count("pcompiler")) { + if (vm["noasm"].as()) + conf::Debug::dumpPexAsm = false; } if (conf::Papyrus::game != GameID::Skyrim) { @@ -489,17 +549,6 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } } - if (vm.count("pcompiler")) { - conf::PCompiler::pCompilerCompatibilityMode = vm["pcompiler"].as(); - conf::PCompiler::all = vm["all"].as(); - conf::PCompiler::norecurse = vm["norecurse"].as(); - if (conf::PCompiler::norecurse) - iterateCompiledDirectoriesRecursively = false; - if (vm["noasm"].as()) - conf::Debug::dumpPexAsm = false; - } - - if (vm.count("import")) { auto dirs = vm["import"].as>(); conf::Papyrus::importDirectories.reserve(dirs.size()); @@ -513,7 +562,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << "Unable to find the import directory '" << sd << "'!" << std::endl; return false; } - conf::Papyrus::importDirectories.push_back(caprica::FSUtils::canonical(sd)); + conf::Papyrus::importDirectories.push_back(FSUtils::canonical(sd)); if (!conf::General::quietCompile) std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back() << std::endl; } @@ -523,59 +572,50 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << "Unable to find the import directory '" << d << "'!" << std::endl; return false; } - conf::Papyrus::importDirectories.push_back(caprica::FSUtils::canonical(d)); + conf::Papyrus::importDirectories.push_back(FSUtils::canonical(d)); if (!conf::General::quietCompile) std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back() << std::endl; } } - std::string baseOutputDir = vm["output"].as(); if (!filesystem::exists(baseOutputDir)) filesystem::create_directories(baseOutputDir); baseOutputDir = FSUtils::canonical(baseOutputDir); + conf::General::outputDirectory = baseOutputDir; - if (vm.count("flags")) { - auto flags = vm["flags"].as(); + std::string flagsPath; + if (!flags.empty()) { + flagsPath = findFlags(flags, progamBasePath, baseOutputDir); + } else if (conf::Papyrus::game == GameID::Starfield) { + std::cout << "No flags specified, Using default Starfield flags file." << std::endl; + flagsPath = "fake://Starfield/Starfield_Papyrus_Flags.flg"; + } - auto flagsPath = findFlags(flags, progamBasePath, baseOutputDir); - if (flagsPath == "") { + if (flagsPath == "") { + if (conf::Papyrus::game != GameID::Starfield) { std::cout << "Unable to locate flags file '" << flags << "'." << std::endl; return false; } - - parseUserFlags(std::move(flagsPath)); - } else if (conf::Papyrus::game == GameID::Starfield) { - std::cout << "No flags specified, Using default Starfield flags file." << std::endl; - parseUserFlags("fake://Starfield/Starfield_Papyrus_Flags.flg"); + std::cout << "Could not find flags file, using default Starfield flags..." << std::endl; + flagsPath = "fake://Starfield/Starfield_Papyrus_Flags.flg"; } + parseUserFlags(std::move(flagsPath)); if (!handleImports(conf::Papyrus::importDirectories, jobManager)) { std::cout << "Import failed!" << std::endl; return false; } - auto filesPassed = std::vector(); - for (auto& f : vm["input-file"].as>()) { - if (f.find(';') != std::string::npos) { - std::istringstream s(f); - std::string sd; - while (getline(s, sd, ';')) - filesPassed.push_back(sd); - } else { - filesPassed.push_back(f); - } - } if (!vm["write-config-file"].as().empty()) SaveDefaultConfig(commandLineDesc, vm["write-config-file"].as(), vm); + + conf::General::inputFiles.reserve(conf::General::inputFiles.size() + filesPassed.size()); + // PCompiler input resolution if (conf::PCompiler::pCompilerCompatibilityMode) { for (auto& f : filesPassed) { if (conf::PCompiler::all) { - if (!addFilesFromDirectory(f, - iterateCompiledDirectoriesRecursively, - baseOutputDir, - jobManager, - caprica::papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, - "")) { + if (!std::filesystem::is_directory(f)) { + std::cout << "Unable to locate input directory '" << f << "'." << std::endl; return false; } } else { @@ -583,40 +623,61 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::replace(f.begin(), f.end(), ':', '\\'); if (FSUtils::extensionAsRef(f).empty()) f.append(".psc"); - auto oDir = baseOutputDir; - if (!addSingleFile(f, oDir, jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile)) + } + auto input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); + if (!std::filesystem::exists(input.resolved_absolute())) { + std::cout << "Unable to locate input file '" << f << "'." << std::endl; + return false; + } + } + } else { + // normal resolution + for (auto& f : filesPassed) { + if (!filesystem::is_directory(f)) { + std::string_view ext = FSUtils::extensionAsRef(f); + if (!pathEq(ext, ".psc") && !pathEq(ext, ".pas") && !pathEq(ext, ".pex") && !pathEq(ext, ".ppj")) { + std::cout << "Don't know how to handle input file '" << f << "'!" << std::endl; + std::cout << "Expected either a Papyrus file (*.psc), Pex assembly file (*.pas), or a Pex file (*.pex)!" + << std::endl; return false; + } + } + auto input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); + if (!std::filesystem::exists(input.resolved_absolute())) { + std::cout << "Unable to locate input file '" << f << "'." << std::endl; + return false; } } - return true; } - // normal resolution - for (auto& f : filesPassed) { - if (!filesystem::exists(f)) { - std::cout << "Unable to locate input file '" << f << "'." << std::endl; - return false; - } - if (filesystem::is_directory(f)) { - if (!addFilesFromDirectory(f, - iterateCompiledDirectoriesRecursively, + for (auto& input : conf::General::inputFiles) { + auto absPath = input.resolved_absolute(); + if (std::filesystem::is_directory(absPath)) { + auto startingDir = input.resolved_relative().string(); + if (startingDir == ".") + startingDir = ""; + auto inputDir = input.resolved_absolute_basedir().string(); + // auto startingNS = input.resolved_relative().string(); + // // replace all `\` with `:` + // std::replace(startingNS.begin(), startingNS.end(), '\\', ':'); + // if (startingNS[0] == ':') + // startingNS = startingNS.substr(1); + if (!addFilesFromDirectory(inputDir, + !input.noRecurse, baseOutputDir, jobManager, - caprica::papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, + papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, + startingDir, "")) { + std::cout << "Unable to add input directory '" << absPath << "'." << std::endl; return false; } - } else { - std::string_view ext = FSUtils::extensionAsRef(f); - if (!pathEq(ext, ".psc") && !pathEq(ext, ".pas") && !pathEq(ext, ".pex") && !pathEq(ext, ".ppj")) { - std::cout << "Don't know how to handle input file '" << f << "'!" << std::endl; - std::cout << "Expected either a Papyrus file (*.psc), Pex assembly file (*.pas), or a Pex file (*.pex)!" - << std::endl; - return false; - } - auto oDir = baseOutputDir; - if (!addSingleFile(f, oDir, jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile)) - return false; + } else if (!addSingleFile(input, + baseOutputDir, + jobManager, + papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile)) { + std::cout << "Unable to add input file '" << absPath << "'." << std::endl; + return false; } } } catch (const std::exception& ex) { From 48e320c0f0d174bc70a17840cb842a971c0cd786 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:51:46 -0700 Subject: [PATCH 06/22] refactor addFilesFromDirectory() --- Caprica/common/CapricaConfig.cpp | 52 +++++++++++++++++++++++---- Caprica/common/CapricaConfig.h | 49 ++++++++++++------------- Caprica/main.cpp | 28 +++++++++------ Caprica/main_options.cpp | 61 +++++++++++++------------------- 4 files changed, 111 insertions(+), 79 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index 97e2507..b9055c3 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -59,7 +59,7 @@ bool allowDecompiledStructNameRefs{ false }; bool enableLanguageExtensions{ false }; bool ignorePropertyNameLocalConflicts{ false }; bool allowImplicitNoneCastsToAnyType{ false }; - std::vector importDirectories{ }; + std::vector importDirectories {}; CapricaUserFlagsDefinition userFlagsDefinition{ }; } @@ -89,24 +89,24 @@ namespace Warnings { std::filesystem::path InputFile::resolved_relative() const { for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir)) - return get_relative_path(dir); + if (dirContains(dir.resolved_absolute())) + return get_relative_path(dir.resolved_absolute()); return {}; // not found } std::filesystem::path InputFile::resolved_absolute() const { // find the file among the import directories for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir)) - return get_absolute_path(dir); + if (dirContains(dir.resolved_absolute())) + return get_absolute_path(dir.resolved_absolute()); return {}; // not found } std::filesystem::path InputFile::resolved_absolute_basedir() const { // find the file among the import directories for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir)) - return dir; + if (dirContains(dir.resolved_absolute())) + return dir.resolved_absolute(); return {}; // not found } @@ -138,4 +138,42 @@ namespace Warnings { } return false; } + InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : noRecurse(noRecurse), + path(std::move(_path)), + cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { + + // special handler; this points to something relative to the cwd, not an object path to be resolved + if (path.string().starts_with(".\\") || path.string().starts_with("./") || path.string().contains("..")) { + if (!path.is_absolute()) + path = cwd / path; + path = FSUtils::canonical(path.string()); + } + } + + bool InputFile::exists() const { + return std::filesystem::exists(resolved_absolute()); + } + + ImportFile::ImportFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : InputFile(_path, noRecurse, _cwd) { + import = true; + // make the import path absolute + if (!path.is_absolute()) + path = cwd / path; + path = FSUtils::canonical(path.string()); + } + + std::filesystem::path ImportFile::resolved_relative() const { + return {}; + } + + std::filesystem::path ImportFile::resolved_absolute() const { + return path; // we always return the absolute path for imports + } + + std::filesystem::path ImportFile::resolved_absolute_basedir() const { + return path; + } + }} diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index c48a2a7..3dbc081 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -13,36 +13,37 @@ namespace caprica { namespace conf { struct InputFile { + virtual std::filesystem::path resolved_relative() const; + virtual std::filesystem::path resolved_absolute() const; + virtual std::filesystem::path resolved_absolute_basedir() const; + std::filesystem::path get_unresolved_path() const { return path; } + bool isRecursive() const { return !noRecurse; } + bool isImport() const { return import; } + bool exists() const; + InputFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = std::filesystem::current_path()); + +protected: bool noRecurse = true; std::filesystem::path path; std::filesystem::path cwd; bool resolved = false; - std::filesystem::path resolved_relative() const; - std::filesystem::path resolved_absolute() const; - std::filesystem::path resolved_absolute_basedir() const; - InputFile(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = std::filesystem::current_path()) - : noRecurse(noRecurse), - path(std::move(_path)), - cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { - - //special handler; this points to something relative to the cwd, not an object path to be resolved - if (path.string().starts_with(".\\") || path.string().starts_with("./") || path.string().contains("..")) - { - if (!path.is_absolute()){ - path = cwd / path; - } - path = FSUtils::canonical(path.string()); - } - } - - private: - std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; - std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; + bool import = false; + virtual std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; + virtual std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; bool dirContains(const std::filesystem::path& dir) const; }; +struct ImportFile : public InputFile { + ImportFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = std::filesystem::current_path()); + virtual std::filesystem::path resolved_relative() const override; + virtual std::filesystem::path resolved_absolute() const override; + virtual std::filesystem::path resolved_absolute_basedir() const override; +}; + // Options that don't fit in any other category. namespace General { // If true, when compiling multiple files, do so @@ -143,7 +144,7 @@ namespace Papyrus { extern bool allowImplicitNoneCastsToAnyType; // The directories to search in for imported types and // unknown types. - extern std::vector importDirectories; + extern std::vector importDirectories; // The user flags definition. extern CapricaUserFlagsDefinition userFlagsDefinition; } diff --git a/Caprica/main.cpp b/Caprica/main.cpp index ea0619a..1f62e54 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -88,8 +88,8 @@ static const std::unordered_set FAKE_SKYRIM_SCRIPTS_SET = { "fake://skyrim/__ScriptObject.psc", "fake://skyrim/DLC1SCWispWallScript.psc", }; -static caseless_unordered_identifier_set abs_import_dirs {}; -bool handleImports(const std::vector &f, caprica::CapricaJobManager *jobManager); + +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType, CapricaJobManager *jobManager, @@ -112,21 +112,29 @@ bool addSingleFile(const conf::InputFile& input, caprica::CapricaJobManager *jobManager, PapyrusCompilationNode::NodeType nodeType); -bool addFilesFromDirectory(const std::string& f, - bool recursive, +bool addFilesFromDirectory(const conf::InputFile& input, const std::string& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType, - const std::string& subdir = "", const std::string& startingNS = "") { // Blargle flargle.... Using the raw Windows API is 5x // faster than boost::filesystem::recursive_directory_iterator, // at 40ms vs. 200ms for the boost solution, and the raw API // solution also gets us the relative paths, absolute paths, // last write time, and file size, all without any extra processing. - auto absBaseDir = caprica::FSUtils::canonical(f); + if (!input.exists()) { + std::cout << "ERROR: Input file '" << input.get_unresolved_path() << "' was not found!" << std::endl; + return false; + } + auto absBaseDir = input.resolved_absolute_basedir().string() + "\\"; + auto subdir = input.resolved_relative().string(); + if (subdir == "." || subdir.empty()) + subdir = "\\"; + else if (subdir[0] != '\\') + subdir = "\\" + subdir; + bool recursive = input.isRecursive(); std::vector dirs{}; - dirs.push_back(subdir.empty() ? "\\" : "\\" + subdir); + dirs.push_back(subdir); auto baseDirMap = getBaseSigMap(conf::Papyrus::game); auto l_startNS = startingNS; while (dirs.size()) { @@ -291,7 +299,7 @@ PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType return getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data.cFileName, lastModTime, fileSize); } -bool handleImports(const std::vector &f, caprica::CapricaJobManager *jobManager) { +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { // Skyrim hacks; we need to import Skyrim's fake scripts into the global namespace first. if (conf::Papyrus::game == GameID::Skyrim) { caprica::caseless_unordered_identifier_ref_map tempMap{}; @@ -312,14 +320,12 @@ bool handleImports(const std::vector &f, caprica::CapricaJobManager } std::cout << "Importing files..." << std::endl; int i = 0; - abs_import_dirs.reserve(f.size()); for (auto& dir : f) { std::string ns = ""; if (conf::Papyrus::game > GameID::Skyrim) ns = "!!temp" + std::to_string(i++); - if (!addFilesFromDirectory(dir, true, "", jobManager, PapyrusCompilationNode::NodeType::PapyrusImport, "", ns)) + if (!addFilesFromDirectory(dir, "", jobManager, PapyrusCompilationNode::NodeType::PapyrusImport, ns)) return false; - abs_import_dirs.emplace(std::filesystem::absolute(dir).string()); } CapricaStats::outputImportedCount(); caprica::papyrus::PapyrusCompilationContext::RenameImports(jobManager); diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index ab13210..ad83a55 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -21,15 +21,13 @@ namespace filesystem = std::filesystem; namespace caprica { struct CapricaJobManager; -bool addFilesFromDirectory(const std::string& f, - bool recursive, +bool addFilesFromDirectory(const conf::InputFile& input, const std::string& baseOutputDir, caprica::CapricaJobManager* jobManager, papyrus::PapyrusCompilationNode::NodeType nodeType, - const std::string& subdir = "", const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); bool addSingleFile(const conf::InputFile& input, const std::string& baseOutputDir, caprica::CapricaJobManager* jobManager, @@ -111,8 +109,8 @@ findFlags(const std::string& flagsPath, const std::string& progamBasePath, const return flagsPath; for (auto& i : conf::Papyrus::importDirectories) - if (filesystem::exists(i + "\\" + flagsPath)) - return i + "\\" + flagsPath; + if (filesystem::exists(i.resolved_absolute() / flagsPath)) + return (i.resolved_absolute() / flagsPath).string(); if (filesystem::exists(baseOutputDir + "\\" + flagsPath)) return baseOutputDir + "\\" + flagsPath; @@ -406,12 +404,8 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } conf::Papyrus::importDirectories.reserve(conf::Papyrus::importDirectories.size() + ppj.imports.size()); - for (auto& i : ppj.imports) { - std::filesystem::path ipath = i; - if (!ipath.is_absolute()) - ipath = (baseDir / i); - conf::Papyrus::importDirectories.emplace_back(FSUtils::canonical(ipath.string())); - } + for (auto& i : ppj.imports) + conf::Papyrus::importDirectories.emplace_back(i, false, baseDir); setPPJBool(ppj.anonymize, conf::General::anonymizeOutput); setPPJBool(ppj.release, conf::CodeGeneration::disableDebugCode); setPPJBool(ppj.finalAttr, conf::CodeGeneration::disableBetaCode); @@ -442,11 +436,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } else { // if we're not doing a project file... // ensure cwd is placed first if (!vm["ignorecwd"].as()) { - conf::Papyrus::importDirectories.reserve(1); - conf::Papyrus::importDirectories.emplace_back(filesystem::current_path().string()); + conf::Papyrus::importDirectories.reserve(conf::Papyrus::importDirectories.size() + 1); + conf::Papyrus::importDirectories.emplace_back(filesystem::current_path(), false); if (!conf::General::quietCompile) { - std::cout << "Adding current working directory to import list: " << conf::Papyrus::importDirectories[0] - << std::endl; + std::cout << "Adding current working directory to import list: " + << conf::Papyrus::importDirectories[0].get_unresolved_path() << std::endl; } } } @@ -551,7 +545,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (vm.count("import")) { auto dirs = vm["import"].as>(); - conf::Papyrus::importDirectories.reserve(dirs.size()); + conf::Papyrus::importDirectories.reserve(conf::Papyrus::importDirectories.size() + dirs.size()); for (auto& d : dirs) { // check if string contains `;` if (d.find(';') != std::string::npos) { @@ -562,9 +556,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << "Unable to find the import directory '" << sd << "'!" << std::endl; return false; } - conf::Papyrus::importDirectories.push_back(FSUtils::canonical(sd)); - if (!conf::General::quietCompile) - std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back() << std::endl; + conf::Papyrus::importDirectories.emplace_back(FSUtils::canonical(sd), false); + if (!conf::General::quietCompile) { + std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back().get_unresolved_path() + << std::endl; + } } continue; } @@ -572,9 +568,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::cout << "Unable to find the import directory '" << d << "'!" << std::endl; return false; } - conf::Papyrus::importDirectories.push_back(FSUtils::canonical(d)); - if (!conf::General::quietCompile) - std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back() << std::endl; + conf::Papyrus::importDirectories.emplace_back(FSUtils::canonical(d), false); + if (!conf::General::quietCompile) { + std::cout << "Adding import directory: " << conf::Papyrus::importDirectories.back().get_unresolved_path() + << std::endl; + } } } @@ -591,7 +589,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage flagsPath = "fake://Starfield/Starfield_Papyrus_Flags.flg"; } - if (flagsPath == "") { + if (flagsPath.empty()) { if (conf::Papyrus::game != GameID::Starfield) { std::cout << "Unable to locate flags file '" << flags << "'." << std::endl; return false; @@ -642,7 +640,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage return false; } } - auto input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); + auto& input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); if (!std::filesystem::exists(input.resolved_absolute())) { std::cout << "Unable to locate input file '" << f << "'." << std::endl; return false; @@ -653,21 +651,10 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage for (auto& input : conf::General::inputFiles) { auto absPath = input.resolved_absolute(); if (std::filesystem::is_directory(absPath)) { - auto startingDir = input.resolved_relative().string(); - if (startingDir == ".") - startingDir = ""; - auto inputDir = input.resolved_absolute_basedir().string(); - // auto startingNS = input.resolved_relative().string(); - // // replace all `\` with `:` - // std::replace(startingNS.begin(), startingNS.end(), '\\', ':'); - // if (startingNS[0] == ':') - // startingNS = startingNS.substr(1); - if (!addFilesFromDirectory(inputDir, - !input.noRecurse, + if (!addFilesFromDirectory(input, baseOutputDir, jobManager, papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, - startingDir, "")) { std::cout << "Unable to add input directory '" << absPath << "'." << std::endl; return false; From 1fd0b730a81f3cc4ea19d9080f176937ccad31fd Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:47:56 -0700 Subject: [PATCH 07/22] fix input directory recursion --- Caprica/main_options.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index ad83a55..02c7e3f 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -622,7 +622,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (FSUtils::extensionAsRef(f).empty()) f.append(".psc"); } - auto input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); + auto input = conf::General::inputFiles.emplace_back(f, !iterateCompiledDirectoriesRecursively); if (!std::filesystem::exists(input.resolved_absolute())) { std::cout << "Unable to locate input file '" << f << "'." << std::endl; return false; @@ -640,7 +640,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage return false; } } - auto& input = conf::General::inputFiles.emplace_back(f, iterateCompiledDirectoriesRecursively); + auto& input = conf::General::inputFiles.emplace_back(f, !iterateCompiledDirectoriesRecursively); if (!std::filesystem::exists(input.resolved_absolute())) { std::cout << "Unable to locate input file '" << f << "'." << std::endl; return false; From 7aa6e39ea2680224928e1189ffd19f9fdfd1fb64 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:49:22 -0700 Subject: [PATCH 08/22] Use filesystem::path in most places in frontend Backend compiler still uses strings --- Caprica/common/CapricaConfig.cpp | 19 ++--- Caprica/common/CapricaConfig.h | 2 +- Caprica/common/FSUtils.cpp | 18 ++++- Caprica/common/FSUtils.h | 2 + Caprica/main.cpp | 128 ++++++++++++++----------------- Caprica/main_options.cpp | 48 ++++++------ 6 files changed, 111 insertions(+), 106 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index b9055c3..63c4518 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -11,7 +11,7 @@ namespace General { bool compileInParallel{ false }; bool quietCompile{ false }; bool recursive { false }; - std::string outputDirectory; + std::filesystem::path outputDirectory; bool anonymizeOutput; std::vector inputFiles; } @@ -112,17 +112,17 @@ namespace Warnings { std::filesystem::path InputFile::get_absolute_path(const std::filesystem::path& absDir) const { if (path.is_absolute()) - return FSUtils::canonical(path.string()); + return FSUtils::canonicalFS(path); else - return FSUtils::canonical((absDir / path).string()); + return FSUtils::canonicalFS((absDir / path)); } std::filesystem::path InputFile::get_relative_path(const std::filesystem::path& absDir) const { std::filesystem::path cpath; if (path.is_absolute()) - cpath = FSUtils::canonical(path.string()); + cpath = FSUtils::canonicalFS(path); else - cpath = FSUtils::canonical((absDir / path).string()); + cpath = FSUtils::canonicalFS((absDir / path)); return cpath.lexically_relative(absDir); } @@ -142,12 +142,13 @@ namespace Warnings { : noRecurse(noRecurse), path(std::move(_path)), cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { - + path = path.make_preferred(); // special handler; this points to something relative to the cwd, not an object path to be resolved - if (path.string().starts_with(".\\") || path.string().starts_with("./") || path.string().contains("..")) { + auto str = path.string(); + if (str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || str.contains("../")) { if (!path.is_absolute()) path = cwd / path; - path = FSUtils::canonical(path.string()); + path = FSUtils::canonicalFS(path); } } @@ -161,7 +162,7 @@ namespace Warnings { // make the import path absolute if (!path.is_absolute()) path = cwd / path; - path = FSUtils::canonical(path.string()); + path = FSUtils::canonicalFS(path); } std::filesystem::path ImportFile::resolved_relative() const { diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index 3dbc081..e3369d6 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -54,7 +54,7 @@ namespace General { // If true, recurse into subdirectories when compiling. extern bool recursive; // self-explanatory - extern std::string outputDirectory; + extern std::filesystem::path outputDirectory; // If true, remove identifying information from the header. extern bool anonymizeOutput; // input files diff --git a/Caprica/common/FSUtils.cpp b/Caprica/common/FSUtils.cpp index 0ab5eb0..b1a0884 100644 --- a/Caprica/common/FSUtils.cpp +++ b/Caprica/common/FSUtils.cpp @@ -46,15 +46,25 @@ std::string_view parentPathAsRef(std::string_view file) { return file; } -// Borrowed and modified from http://stackoverflow.com/a/1750710/776797 -std::string canonical(const std::string& path) { +bool shouldShortCircuit(const std::string& path) { if (path.size() > 3 && path[1] == ':') { // Shortcircuit for already canon paths. if (path.find("/") == std::string::npos && path.find("..") == std::string::npos && path.find("\\.\\") == std::string::npos && path.find("\\\\") == std::string::npos) { - return path; + return true; } } + return false; +} + +// Borrowed and modified from http://stackoverflow.com/a/1750710/776797 +std::string canonical(const std::string& path) { + return canonicalFS(path).string(); +} + +std::filesystem::path canonicalFS(const std::filesystem::path& path) { + if (shouldShortCircuit(path.string())) + return path; auto absPath = std::filesystem::absolute(path); if (conf::Performance::resolveSymlinks) { return std::filesystem::canonical(absPath).make_preferred().string(); @@ -76,7 +86,7 @@ std::string canonical(const std::string& path) { result /= *it; } } - return result.make_preferred().string(); + return result.make_preferred(); } } diff --git a/Caprica/common/FSUtils.h b/Caprica/common/FSUtils.h index bfd8394..b4849d7 100644 --- a/Caprica/common/FSUtils.h +++ b/Caprica/common/FSUtils.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -14,5 +15,6 @@ std::string_view filenameAsRef(std::string_view file); std::string_view parentPathAsRef(std::string_view file); std::string canonical(const std::string& path); +std::filesystem::path canonicalFS(const std::filesystem::path& path); }} diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 1f62e54..5494100 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -91,29 +91,29 @@ static const std::unordered_set FAKE_SKYRIM_SCRIPTS_SET = { bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); -PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType, - CapricaJobManager *jobManager, - const std::string &baseOutputDir, - const std::string &curDir, - const std::string &absBaseDir, - const WIN32_FIND_DATA &fileName); - -PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType, - CapricaJobManager *jobManager, - const std::string &baseOutputDir, - const std::string &curDir, - const std::string &absBaseDir, - const std::string &fileName, +PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, + CapricaJobManager* jobManager, + const std::filesystem::path& baseOutputDir, + const std::filesystem::path& curDir, + const std::filesystem::path& absBaseDir, + const WIN32_FIND_DATA& data); + +PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, + CapricaJobManager* jobManager, + const std::filesystem::path& baseOutputDir, + const std::filesystem::path& curDir, + const std::filesystem::path& absBaseDir, + const std::filesystem::path& fileName, time_t lastModTime, size_t fileSize); bool addSingleFile(const conf::InputFile& input, - const std::string &baseOutputDir, + const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager *jobManager, PapyrusCompilationNode::NodeType nodeType); bool addFilesFromDirectory(const conf::InputFile& input, - const std::string& baseOutputDir, + const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType, const std::string& startingNS = "") { @@ -126,27 +126,28 @@ bool addFilesFromDirectory(const conf::InputFile& input, std::cout << "ERROR: Input file '" << input.get_unresolved_path() << "' was not found!" << std::endl; return false; } - auto absBaseDir = input.resolved_absolute_basedir().string() + "\\"; - auto subdir = input.resolved_relative().string(); - if (subdir == "." || subdir.empty()) - subdir = "\\"; - else if (subdir[0] != '\\') - subdir = "\\" + subdir; + auto absBaseDir = input.resolved_absolute_basedir(); + auto subdir = input.resolved_relative(); + if (subdir == ".") + subdir = ""; bool recursive = input.isRecursive(); - std::vector dirs{}; + std::vector dirs {}; dirs.push_back(subdir); auto baseDirMap = getBaseSigMap(conf::Papyrus::game); auto l_startNS = startingNS; + const auto DOTDOT = std::string_view(".."); + const auto DOT = std::string_view("."); + while (dirs.size()) { HANDLE hFind; WIN32_FIND_DATA data; auto curDir = dirs.back(); dirs.pop_back(); - auto curSearchPattern = absBaseDir + curDir + "\\*"; + auto curSearchPattern = absBaseDir / curDir / "*"; caprica::caseless_unordered_identifier_ref_map namespaceMap{}; namespaceMap.reserve(8000); - hFind = FindFirstFileA(curSearchPattern.c_str(), &data); + hFind = FindFirstFileA(curSearchPattern.string().c_str(), &data); if (hFind == INVALID_HANDLE_VALUE) { std::cout << "An error occurred while trying to iterate the files in '" << curSearchPattern << "'!" << std::endl; return false; @@ -154,14 +155,10 @@ bool addFilesFromDirectory(const conf::InputFile& input, do { std::string_view filenameRef = data.cFileName; - if (filenameRef != std::string_view(".") && filenameRef != std::string_view("..")) { + if (filenameRef != DOT && filenameRef != DOTDOT) { if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - if (recursive) { - if (curDir == "\\") - dirs.push_back(curDir + data.cFileName); - else - dirs.push_back(curDir + "\\" + data.cFileName); - } + if (recursive) + dirs.push_back(curDir / data.cFileName); } else { auto ext = FSUtils::extensionAsRef(filenameRef); bool skip = false; @@ -187,7 +184,7 @@ bool addFilesFromDirectory(const conf::InputFile& input, break; } if (!skip) { - PapyrusCompilationNode *node = getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data); + PapyrusCompilationNode* node = getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data); namespaceMap.emplace(caprica::identifier_ref(node->baseName), node); if (!gBaseFound && !baseDirMap.empty()) { @@ -220,7 +217,7 @@ bool addFilesFromDirectory(const conf::InputFile& input, } } // if it's true, then this is the base dir and the namespace should be root - auto namespaceName = l_startNS + curDir; + auto namespaceName = l_startNS + curDir.lexically_normal().string(); std::replace(namespaceName.begin(), namespaceName.end(), '\\', ':'); while (namespaceName[0] == ':') namespaceName = namespaceName.substr(1); @@ -235,32 +232,23 @@ bool addFilesFromDirectory(const conf::InputFile& input, return true; } -PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType, - CapricaJobManager *jobManager, - const std::string &baseOutputDir, - const std::string &curDir, - const std::string &absBaseDir, - const std::string &fileName, +PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, + CapricaJobManager* jobManager, + const std::filesystem::path& baseOutputDir, + const std::filesystem::path& curDir, + const std::filesystem::path& absBaseDir, + const std::filesystem::path& fileName, time_t lastModTime, size_t fileSize) { - std::string curDirFull; - if (curDir == "\\" || curDir == "") - curDirFull = absBaseDir; + std::filesystem::path cur; + if (curDir == ".") + cur = ""; else - curDirFull = absBaseDir + curDir; - - std::string sourceFilePath = curDirFull + "\\" + fileName; - std::string filenameToDisplay; - std::string outputDir; - if (curDir == "\\" || curDir == "") { - filenameToDisplay = fileName; - outputDir = baseOutputDir; - } else { - filenameToDisplay = curDir + "\\" + fileName; - if (filenameToDisplay[0] == '\\') - filenameToDisplay = filenameToDisplay.substr(1); - outputDir = baseOutputDir + curDir; - } + cur = curDir; + std::filesystem::path sourceFilePath = absBaseDir / cur / fileName; + std::filesystem::path filenameToDisplay = cur / fileName; + std::filesystem::path outputDir = baseOutputDir / cur; + if (nodeType == PapyrusCompilationNode::NodeType::PapyrusImport || nodeType == PapyrusCompilationNode::NodeType::PasReflection || nodeType == PapyrusCompilationNode::NodeType::PexReflection) { @@ -270,20 +258,20 @@ PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType } auto node = new caprica::papyrus::PapyrusCompilationNode(jobManager, nodeType, - std::move(filenameToDisplay), - std::move(outputDir), - std::move(sourceFilePath), + std::move(filenameToDisplay.string()), + std::move(outputDir.string()), + std::move(sourceFilePath.string()), lastModTime, fileSize); return node; } -PapyrusCompilationNode *getNode(const PapyrusCompilationNode::NodeType &nodeType, - CapricaJobManager *jobManager, - const std::string &baseOutputDir, - const std::string &curDir, - const std::string &absBaseDir, - const WIN32_FIND_DATA &data) { +PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, + CapricaJobManager* jobManager, + const std::filesystem::path& baseOutputDir, + const std::filesystem::path& curDir, + const std::filesystem::path& absBaseDir, + const WIN32_FIND_DATA& data) { const auto lastModTime = [](FILETIME ft) -> time_t { ULARGE_INTEGER ull; ull.LowPart = ft.dwLowDateTime; @@ -333,18 +321,18 @@ bool handleImports(const std::vector& f, caprica::CapricaJobMa } bool addSingleFile(const conf::InputFile& input, - const std::string& baseOutputDir, + const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType) { // Get the file size and last modified time using std::filesystem std::error_code ec; auto relPath = input.resolved_relative(); - std::string namespaceDir = relPath.parent_path().string(); + auto namespaceDir = relPath.parent_path(); auto absPath = input.resolved_absolute(); - auto filename = absPath.filename().string(); - std::string absBaseDir = input.resolved_absolute_basedir().string() + "\\"; - if (!std::filesystem::exists(absPath)) { + auto filename = absPath.filename(); + auto absBaseDir = input.resolved_absolute_basedir(); + if (!input.exists()) { std::cout << "ERROR: Resolved file path '" << absPath << "' is not in an import directory, cannot resolve namespace!" << std::endl; return false; diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index 02c7e3f..fa8c915 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -22,14 +22,14 @@ namespace caprica { struct CapricaJobManager; bool addFilesFromDirectory(const conf::InputFile& input, - const std::string& baseOutputDir, + const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, papyrus::PapyrusCompilationNode::NodeType nodeType, const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); bool addSingleFile(const conf::InputFile& input, - const std::string& baseOutputDir, + const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType nodeType); @@ -103,21 +103,22 @@ void SaveDefaultConfig(const po::options_description& descOptions, boost::property_tree::write_ini(configFile, tree); } -std::string -findFlags(const std::string& flagsPath, const std::string& progamBasePath, const std::string& baseOutputDir) { +std::string findFlags(const std::filesystem::path& flagsPath, + const std::filesystem::path& progamBasePath, + const std::filesystem::path& baseOutputDir) { if (filesystem::exists(flagsPath)) - return flagsPath; + return flagsPath.string(); for (auto& i : conf::Papyrus::importDirectories) if (filesystem::exists(i.resolved_absolute() / flagsPath)) return (i.resolved_absolute() / flagsPath).string(); - if (filesystem::exists(baseOutputDir + "\\" + flagsPath)) - return baseOutputDir + "\\" + flagsPath; - if (filesystem::exists(progamBasePath + "\\" + flagsPath)) - return progamBasePath + "\\" + flagsPath; + if (filesystem::exists(baseOutputDir / flagsPath)) + return (baseOutputDir / flagsPath).string(); + if (filesystem::exists(progamBasePath / flagsPath)) + return (progamBasePath / flagsPath).string(); - return ""; + return {}; }; void setPPJBool(BoolSetting value, bool& setting) { @@ -307,11 +308,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage vm); po::notify(vm); std::string confFilePath = vm["config-file"].as(); - auto progamBasePath = filesystem::absolute(filesystem::path(argv[0]).parent_path()).string(); + auto progamBasePath = filesystem::absolute(filesystem::path(argv[0]).parent_path()); bool loadedConfigFile = false; - if (filesystem::exists(progamBasePath + "\\" + confFilePath)) { + if (filesystem::exists(progamBasePath / confFilePath)) { loadedConfigFile = true; - std::ifstream ifs(progamBasePath + "\\" + confFilePath); + std::ifstream ifs(progamBasePath / confFilePath); auto config_opts = po::parse_config_file(ifs, commandLineDesc); po::store(config_opts, vm); po::notify(vm); @@ -358,7 +359,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } std::string flags; - std::string baseOutputDir; + std::filesystem::path baseOutputDir; // we have to put this at the beginning, because flags passed override this PapyrusProject ppj; std::string ppjPath; @@ -394,7 +395,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (!ppjPath.empty()) { auto baseDir = filesystem::path(ppjPath).parent_path(); if (!baseDir.empty()) - baseDir = FSUtils::canonical(baseDir.string()); + baseDir = FSUtils::canonicalFS(baseDir); else baseDir = filesystem::current_path(); // set all the options in the ppj @@ -425,7 +426,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage auto outputDir = std::filesystem::path(ppj.output); if (outputDir.is_relative()) outputDir = (baseDir / ppj.output); - baseOutputDir = outputDir.string(); + baseOutputDir = outputDir; conf::General::inputFiles.reserve(conf::General::inputFiles.size() + ppj.folders.size() + ppj.scripts.size()); for (auto& f : ppj.folders) @@ -469,7 +470,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (vm.count("output")) baseOutputDir = vm["output"].as(); else if (baseOutputDir.empty()) - baseOutputDir = filesystem::current_path().string(); + baseOutputDir = filesystem::current_path(); if (vm.count("pcompiler")) { if (vm["noasm"].as()) conf::Debug::dumpPexAsm = false; @@ -578,7 +579,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (!filesystem::exists(baseOutputDir)) filesystem::create_directories(baseOutputDir); - baseOutputDir = FSUtils::canonical(baseOutputDir); + baseOutputDir = FSUtils::canonicalFS(baseOutputDir); conf::General::outputDirectory = baseOutputDir; std::string flagsPath; @@ -649,21 +650,24 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } for (auto& input : conf::General::inputFiles) { - auto absPath = input.resolved_absolute(); - if (std::filesystem::is_directory(absPath)) { + if (!input.exists()) { + std::cout << "Unable to locate input file '" << input.get_unresolved_path() << "'." << std::endl; + return false; + } + if (std::filesystem::is_directory(input.resolved_absolute())) { if (!addFilesFromDirectory(input, baseOutputDir, jobManager, papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, "")) { - std::cout << "Unable to add input directory '" << absPath << "'." << std::endl; + std::cout << "Unable to add input directory '" << input.get_unresolved_path() << "'." << std::endl; return false; } } else if (!addSingleFile(input, baseOutputDir, jobManager, papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile)) { - std::cout << "Unable to add input file '" << absPath << "'." << std::endl; + std::cout << "Unable to add input file '" << input.get_unresolved_path() << "'." << std::endl; return false; } } From 43c96edbe17421ace2ba49a3deedc6b4640bc8a6 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:57:33 -0700 Subject: [PATCH 09/22] ensure that CLI flags override ppj options --- Caprica/main_options.cpp | 129 ++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 50 deletions(-) diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index fa8c915..cbc6bc2 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -126,6 +126,24 @@ void setPPJBool(BoolSetting value, bool& setting) { setting = value == BoolSetting::True ? true : false; }; +caseless_unordered_identifier_map gameIDMap = { + {"Starfield", GameID::Starfield}, + { "Skyrim", GameID::Skyrim }, + { "Fallout4", GameID::Fallout4 }, + { "Fallout76", GameID::Fallout76}, +}; + +bool setGame(std::string gameType) { + auto it = gameIDMap.find(gameType); + if (it != gameIDMap.end()) { + conf::Papyrus::game = it->second; + return true; + } + return false; +} + +constexpr const char* DEFAULT_GAME = "Starfield"; + bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManager* jobManager) { try { bool iterateCompiledDirectoriesRecursively = false; @@ -142,16 +160,17 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage "Set the file defining the user flags.") ("output,o", po::value(), "Set the directory to save compiler output to.") - ("optimize,op,O", po::bool_switch(&conf::CodeGeneration::enableOptimizations)->default_value(false), + ("optimize,op,O", "Enable optimizations.") ("parallel-compile,p", po::bool_switch(&conf::General::compileInParallel)->default_value(false), "Compile files in parallel.") - ("release,r", po::bool_switch(&conf::CodeGeneration::disableDebugCode)->default_value(false), + ("release,r", "Don't generate DebugOnly code.") ("recurse,R", po::bool_switch(&iterateCompiledDirectoriesRecursively)->default_value(false), "Recursively compile all scripts in the directories passed.") - ("final", po::bool_switch(&conf::CodeGeneration::disableBetaCode)->default_value(false), + ("final", "Don't generate BetaOnly code.") + ("anonymize", "Anonymize script header information.") ("all-warnings-as-errors", po::bool_switch(&conf::Warnings::treatWarningsAsErrors)->default_value(false), "Treat all warnings as if they were errors.") @@ -211,7 +230,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage "Allow async file reading. This is primarily useful on SSDs.") ("async-write", po::value(&conf::Performance::asyncFileWrite)->default_value(true), "Allow writing output to disk on background threads.") - ("dump-asm,keepasm", po::bool_switch(&conf::Debug::dumpPexAsm)->default_value(false), + ("dump-asm,keepasm", "Dump the PEX assembly code for the input files.") ("enable-ck-optimizations", po::value(&conf::CodeGeneration::enableCKOptimizations)->default_value(true), "Enable optimizations that the CK compiler normally does regardless of the -optimize switch.") @@ -245,7 +264,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage "Enable PCompiler compatibility mode (default false).") ("all,a", po::bool_switch(&conf::PCompiler::all)->default_value(false), "Treat input objects as directories") ("norecurse", po::bool_switch(&conf::PCompiler::norecurse)->default_value(false), "Don't recursively scan directories with -all") - ("noasm", po::bool_switch()->default_value(false), "Turns off asm output if it was turned on.") + ("noasm", "Turns off asm output if it was turned on.") ("asmonly", po::bool_switch()->default_value(false), "Does nothing on Caprica") ("debug,d", po::bool_switch()->default_value(false), "Does nothing on Caprica"); @@ -360,7 +379,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::string flags; std::filesystem::path baseOutputDir; - // we have to put this at the beginning, because flags passed override this + PapyrusProject ppj; std::string ppjPath; auto filesToRemove = std::vector(); @@ -392,36 +411,56 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage iterateCompiledDirectoriesRecursively = false; } + // we have to make sure that the game value is set for sure here + // CLI flags override the project file + if (vm.count("game")) { + if (!ppjPath.empty() && ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + std::cout << "Warning: Game type specified in both project file and command line, using command line value '" + << vm["game"].as() << "'." << std::endl; + } + if (!setGame(vm["game"].as())) { + std::cout << "Unrecognized game type '" << vm["game"].as() << "'!" << std::endl; + return false; + } + } else if (!ppjPath.empty() && ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { + conf::Papyrus::game = + ppj.game == PapyrusProject::PapyrusGame::SkyrimSpecialEdition ? GameID::Skyrim : (GameID)ppj.game; + } else { + setGame(DEFAULT_GAME); + } + if (!ppjPath.empty()) { auto baseDir = filesystem::path(ppjPath).parent_path(); if (!baseDir.empty()) baseDir = FSUtils::canonicalFS(baseDir); else baseDir = filesystem::current_path(); - // set all the options in the ppj - if (ppj.game != PapyrusProject::PapyrusGame::UNKNOWN) { - conf::Papyrus::game = - ppj.game == PapyrusProject::PapyrusGame::SkyrimSpecialEdition ? GameID::Skyrim : (GameID)ppj.game; - } conf::Papyrus::importDirectories.reserve(conf::Papyrus::importDirectories.size() + ppj.imports.size()); for (auto& i : ppj.imports) conf::Papyrus::importDirectories.emplace_back(i, false, baseDir); - setPPJBool(ppj.anonymize, conf::General::anonymizeOutput); - setPPJBool(ppj.release, conf::CodeGeneration::disableDebugCode); - setPPJBool(ppj.finalAttr, conf::CodeGeneration::disableBetaCode); - setPPJBool(ppj.optimize, conf::CodeGeneration::enableOptimizations); - switch (ppj.asmAttr) { - case PapyrusProject::AsmType::None: - case PapyrusProject::AsmType::Discard: - conf::Debug::dumpPexAsm = false; - break; - case PapyrusProject::AsmType::Keep: - case PapyrusProject::AsmType::Only: // TODO: HANDLE THIS - conf::Debug::dumpPexAsm = true; - break; - default: - break; + + if (!vm.count("anonymize")) + setPPJBool(ppj.anonymize, conf::General::anonymizeOutput); + if (!vm.count("release")) + setPPJBool(ppj.release, conf::CodeGeneration::disableDebugCode); + if (!vm.count("final")) + setPPJBool(ppj.finalAttr, conf::CodeGeneration::disableBetaCode); + if (!vm.count("optimize")) + setPPJBool(ppj.optimize, conf::CodeGeneration::enableOptimizations); + if (!vm.count("dump-asm") && !vm.count("keep-asm") && !vm.count("noasm")) { + switch (ppj.asmAttr) { + case PapyrusProject::AsmType::None: + case PapyrusProject::AsmType::Discard: + conf::Debug::dumpPexAsm = false; + break; + case PapyrusProject::AsmType::Keep: + case PapyrusProject::AsmType::Only: // TODO: HANDLE THIS + conf::Debug::dumpPexAsm = true; + break; + default: + break; + } } auto outputDir = std::filesystem::path(ppj.output); if (outputDir.is_relative()) @@ -445,25 +484,6 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } } } - if (conf::Papyrus::game == GameID::UNKNOWN) { - std::string gameType; - if (vm.count("game")) - gameType = vm["game"].as(); - else - gameType = "Starfield"; - if (_stricmp(gameType.c_str(), "Starfield") == 0) { - conf::Papyrus::game = GameID::Starfield; - } else if (_stricmp(gameType.c_str(), "Skyrim") == 0) { - conf::Papyrus::game = GameID::Skyrim; - } else if (_stricmp(gameType.c_str(), "Fallout4") == 0) { - conf::Papyrus::game = GameID::Fallout4; - } else if (_stricmp(gameType.c_str(), "Fallout76") == 0) { - conf::Papyrus::game = GameID::Fallout76; - } else { - std::cout << "Unrecognized game type '" << gameType << "'!" << std::endl; - return false; - } - } if (vm.count("flags")) flags = vm["flags"].as(); @@ -471,10 +491,19 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage baseOutputDir = vm["output"].as(); else if (baseOutputDir.empty()) baseOutputDir = filesystem::current_path(); - if (vm.count("pcompiler")) { - if (vm["noasm"].as()) - conf::Debug::dumpPexAsm = false; - } + + if (vm.count("anonymize")) + conf::General::anonymizeOutput = true; + if (vm.count("release")) + conf::CodeGeneration::disableDebugCode = true; + if (vm.count("final")) + conf::CodeGeneration::disableBetaCode = true; + if (vm.count("optimize")) + conf::CodeGeneration::enableOptimizations = true; + if (vm.count("dump-asm") || vm.count("keepasm")) + conf::Debug::dumpPexAsm = true; + if (vm.count("pcompiler") && vm.count("noasm")) + conf::Debug::dumpPexAsm = false; if (conf::Papyrus::game != GameID::Skyrim) { // turn off skyrim options @@ -486,7 +515,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } // TODO: enable this eventually - if (vm["optimize"].as() && conf::Papyrus::game != GameID::Fallout4) { + if (conf::CodeGeneration::enableOptimizations && conf::Papyrus::game != GameID::Fallout4) { if (!vm["force-enable-optimizations"].as()) { conf::CodeGeneration::enableOptimizations = false; std::cout << "Warning: Optimization is currently only supported for Fallout 4, disabling..." << std::endl; From 5642931967c46206bf644f7db3865fc2e7dc2518 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 20:22:23 -0700 Subject: [PATCH 10/22] fix import resolution bug --- Caprica/common/CapricaConfig.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index 63c4518..396f8d4 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -130,7 +130,8 @@ namespace Warnings { if (path.is_absolute()) { // check if the path is contained in the import directory - if (!path.lexically_relative(dir).string().starts_with("..")) + auto rel = path.lexically_relative(dir).string(); + if (!rel.empty() && !rel.starts_with("..")) return true; } else { if (std::filesystem::exists(dir / path)) From 869201968e164277b630aa5c1a30aa9e4a1983ba Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 21:25:58 -0700 Subject: [PATCH 11/22] make ppj game attribute optional --- Caprica/common/parser/CapricaPPJParser.cpp | 185 +-------------------- 1 file changed, 7 insertions(+), 178 deletions(-) diff --git a/Caprica/common/parser/CapricaPPJParser.cpp b/Caprica/common/parser/CapricaPPJParser.cpp index ead3d43..c02ac25 100644 --- a/Caprica/common/parser/CapricaPPJParser.cpp +++ b/Caprica/common/parser/CapricaPPJParser.cpp @@ -7,183 +7,6 @@ #include #include namespace caprica { -/** - * PapyrusProject.xsd: - ```xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -``` - */ - -/** - * Parses a PPJ file and returns a PapyrusProject object. - * The PPJ file is an XML file that describes a Papyrus project. - * See PapyrusProject.xsd for the schema. - */ static caseless_unordered_identifier_map GAME_MAP = { {"sf1", PapyrusProject::PapyrusGame::Starfield }, @@ -290,12 +113,13 @@ inline CommandList CapricaPPJParser::ParseCommandList(const pugi::xml_node& node inline PapyrusProject::PapyrusGame ParseGameAttribute(const pugi::xml_node& root) { std::string game = root.attribute("Game").as_string(); if (game.empty()) - throw std::runtime_error("PapyrusProject must have a Game attribute!"); + return PapyrusProject::PapyrusGame::NOT_SET; auto it = GAME_MAP.find(game); if (it == GAME_MAP.end()) throw std::runtime_error("PapyrusProject has an invalid Game attribute!"); return it->second; } + inline PapyrusProject::AsmType ParseAsmAttribute(const pugi::xml_node& root) { std::string asmAttr = root.attribute("Asm").as_string(); if (asmAttr.empty()) @@ -321,6 +145,11 @@ bool CapricaPPJParser::ParseVariables(const pugi::xml_node& node) { return true; } +/** + * Parses a PPJ file and returns a PapyrusProject object. + * The PPJ file is an XML file that describes a Papyrus project. + * See PapyrusProject.xsd for the schema. + */ PapyrusProject CapricaPPJParser::Parse(const std::string& path) { pugi::xml_document doc; pugi::xml_parse_result result = doc.load_file(path.c_str()); From 91eaf476ab0469e0e3017aea8d5aecb120780593 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 21:28:16 -0700 Subject: [PATCH 12/22] check for input file if . or .. --- Caprica/common/CapricaConfig.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index 396f8d4..a7612b0 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -53,14 +53,14 @@ namespace EngineLimits { namespace Papyrus { GameID game { GameID::UNKNOWN }; -bool allowCompilerIdentifiers { false }; -bool allowDecompiledStructNameRefs{ false }; + bool allowCompilerIdentifiers { false }; + bool allowDecompiledStructNameRefs{ false }; bool allowNegativeLiteralAsBinaryOp{ false }; bool enableLanguageExtensions{ false }; bool ignorePropertyNameLocalConflicts{ false }; bool allowImplicitNoneCastsToAnyType{ false }; std::vector importDirectories {}; - CapricaUserFlagsDefinition userFlagsDefinition{ }; + CapricaUserFlagsDefinition userFlagsDefinition{}; } namespace Skyrim { @@ -146,7 +146,8 @@ namespace Warnings { path = path.make_preferred(); // special handler; this points to something relative to the cwd, not an object path to be resolved auto str = path.string(); - if (str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || str.contains("../")) { + if (str == "." || str == ".." || str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || + str.contains("../")) { if (!path.is_absolute()) path = cwd / path; path = FSUtils::canonicalFS(path); From eba6a94ec385e9fe919ca9e5fc5580c4e456fc4e Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 21:29:22 -0700 Subject: [PATCH 13/22] dont set default asm attribute --- Caprica/common/parser/CapricaPPJParser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Caprica/common/parser/CapricaPPJParser.cpp b/Caprica/common/parser/CapricaPPJParser.cpp index c02ac25..7f74335 100644 --- a/Caprica/common/parser/CapricaPPJParser.cpp +++ b/Caprica/common/parser/CapricaPPJParser.cpp @@ -123,7 +123,7 @@ inline PapyrusProject::PapyrusGame ParseGameAttribute(const pugi::xml_node& root inline PapyrusProject::AsmType ParseAsmAttribute(const pugi::xml_node& root) { std::string asmAttr = root.attribute("Asm").as_string(); if (asmAttr.empty()) - return PapyrusProject::AsmType::None; + return PapyrusProject::AsmType::NOT_SET; auto it = ASM_MAP.find(asmAttr); if (it == ASM_MAP.end()) throw std::runtime_error("PapyrusProject has an invalid Asm attribute!"); From bf0f2a6eb4d9ad705851016fc4335f2a330f6754 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 22:08:22 -0700 Subject: [PATCH 14/22] move input file to separate file --- Caprica/common/CapricaConfig.cpp | 92 ------------------------- Caprica/common/CapricaConfig.h | 33 +-------- Caprica/common/CapricaInputFile.cpp | 101 ++++++++++++++++++++++++++++ Caprica/common/CapricaInputFile.h | 36 ++++++++++ Caprica/main.cpp | 10 +-- Caprica/main_options.cpp | 6 +- 6 files changed, 146 insertions(+), 132 deletions(-) create mode 100644 Caprica/common/CapricaInputFile.cpp create mode 100644 Caprica/common/CapricaInputFile.h diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index a7612b0..2e89f07 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -87,96 +87,4 @@ namespace Warnings { std::unordered_set warningsToEnable{ }; } - std::filesystem::path InputFile::resolved_relative() const { - for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return get_relative_path(dir.resolved_absolute()); - return {}; // not found - } - - std::filesystem::path InputFile::resolved_absolute() const { - // find the file among the import directories - for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return get_absolute_path(dir.resolved_absolute()); - return {}; // not found - } - - std::filesystem::path InputFile::resolved_absolute_basedir() const { - // find the file among the import directories - for (auto& dir : Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return dir.resolved_absolute(); - return {}; // not found - } - - std::filesystem::path InputFile::get_absolute_path(const std::filesystem::path& absDir) const { - if (path.is_absolute()) - return FSUtils::canonicalFS(path); - else - return FSUtils::canonicalFS((absDir / path)); - } - - std::filesystem::path InputFile::get_relative_path(const std::filesystem::path& absDir) const { - std::filesystem::path cpath; - if (path.is_absolute()) - cpath = FSUtils::canonicalFS(path); - else - cpath = FSUtils::canonicalFS((absDir / path)); - return cpath.lexically_relative(absDir); - } - - bool InputFile::dirContains(const std::filesystem::path& dir) const { - - if (path.is_absolute()) { - // check if the path is contained in the import directory - auto rel = path.lexically_relative(dir).string(); - if (!rel.empty() && !rel.starts_with("..")) - return true; - } else { - if (std::filesystem::exists(dir / path)) - return true; - } - return false; - } - InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : noRecurse(noRecurse), - path(std::move(_path)), - cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { - path = path.make_preferred(); - // special handler; this points to something relative to the cwd, not an object path to be resolved - auto str = path.string(); - if (str == "." || str == ".." || str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || - str.contains("../")) { - if (!path.is_absolute()) - path = cwd / path; - path = FSUtils::canonicalFS(path); - } - } - - bool InputFile::exists() const { - return std::filesystem::exists(resolved_absolute()); - } - - ImportFile::ImportFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : InputFile(_path, noRecurse, _cwd) { - import = true; - // make the import path absolute - if (!path.is_absolute()) - path = cwd / path; - path = FSUtils::canonicalFS(path); - } - - std::filesystem::path ImportFile::resolved_relative() const { - return {}; - } - - std::filesystem::path ImportFile::resolved_absolute() const { - return path; // we always return the absolute path for imports - } - - std::filesystem::path ImportFile::resolved_absolute_basedir() const { - return path; - } - }} diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index e3369d6..6ead0ef 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -10,40 +10,9 @@ #include #include #include +#include namespace caprica { namespace conf { -struct InputFile { - virtual std::filesystem::path resolved_relative() const; - virtual std::filesystem::path resolved_absolute() const; - virtual std::filesystem::path resolved_absolute_basedir() const; - std::filesystem::path get_unresolved_path() const { return path; } - bool isRecursive() const { return !noRecurse; } - bool isImport() const { return import; } - bool exists() const; - InputFile(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = std::filesystem::current_path()); - -protected: - bool noRecurse = true; - std::filesystem::path path; - std::filesystem::path cwd; - bool resolved = false; - bool import = false; - virtual std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; - virtual std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; - bool dirContains(const std::filesystem::path& dir) const; -}; - -struct ImportFile : public InputFile { - ImportFile(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = std::filesystem::current_path()); - virtual std::filesystem::path resolved_relative() const override; - virtual std::filesystem::path resolved_absolute() const override; - virtual std::filesystem::path resolved_absolute_basedir() const override; -}; - // Options that don't fit in any other category. namespace General { // If true, when compiling multiple files, do so diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp new file mode 100644 index 0000000..b1218bb --- /dev/null +++ b/Caprica/common/CapricaInputFile.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include + +namespace caprica{ + + std::filesystem::path InputFile::resolved_relative() const { + for (auto& dir : conf::Papyrus::importDirectories) + if (dirContains(dir.resolved_absolute())) + return get_relative_path(dir.resolved_absolute()); + return {}; // not found + } + + std::filesystem::path InputFile::resolved_absolute() const { + // find the file among the import directories + for (auto& dir : conf::Papyrus::importDirectories) + if (dirContains(dir.resolved_absolute())) + return get_absolute_path(dir.resolved_absolute()); + return {}; // not found + } + + std::filesystem::path InputFile::resolved_absolute_basedir() const { + // find the file among the import directories + for (auto& dir : conf::Papyrus::importDirectories) + if (dirContains(dir.resolved_absolute())) + return dir.resolved_absolute(); + return {}; // not found + } + + std::filesystem::path InputFile::get_absolute_path(const std::filesystem::path& absDir) const { + if (path.is_absolute()) + return FSUtils::canonicalFS(path); + else + return FSUtils::canonicalFS((absDir / path)); + } + + std::filesystem::path InputFile::get_relative_path(const std::filesystem::path& absDir) const { + std::filesystem::path cpath; + if (path.is_absolute()) + cpath = FSUtils::canonicalFS(path); + else + cpath = FSUtils::canonicalFS((absDir / path)); + return cpath.lexically_relative(absDir); + } + + bool InputFile::dirContains(const std::filesystem::path& dir) const { + + if (path.is_absolute()) { + // check if the path is contained in the import directory + auto rel = path.lexically_relative(dir).string(); + if (!rel.empty() && !rel.starts_with("..")) + return true; + } else { + if (std::filesystem::exists(dir / path)) + return true; + } + return false; + } + InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : noRecurse(noRecurse), + path(std::move(_path)), + cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { + path = path.make_preferred(); + // special handler; this points to something relative to the cwd, not an object path to be resolved + auto str = path.string(); + if (str == "." || str == ".." || str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || + str.contains("../")) { + if (!path.is_absolute()) + path = cwd / path; + path = FSUtils::canonicalFS(path); + } + } + + bool InputFile::exists() const { + return std::filesystem::exists(resolved_absolute()); + } + + ImportFile::ImportFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : InputFile(_path, noRecurse, _cwd) { + import = true; + // make the import path absolute + if (!path.is_absolute()) + path = cwd / path; + path = FSUtils::canonicalFS(path); + } + + std::filesystem::path ImportFile::resolved_relative() const { + return {}; + } + + std::filesystem::path ImportFile::resolved_absolute() const { + return path; // we always return the absolute path for imports + } + + std::filesystem::path ImportFile::resolved_absolute_basedir() const { + return path; + } + +} // namespace caprica \ No newline at end of file diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h new file mode 100644 index 0000000..0506e42 --- /dev/null +++ b/Caprica/common/CapricaInputFile.h @@ -0,0 +1,36 @@ +#pragma once +#include +namespace caprica{ + +struct InputFile { + virtual std::filesystem::path resolved_relative() const; + virtual std::filesystem::path resolved_absolute() const; + virtual std::filesystem::path resolved_absolute_basedir() const; + std::filesystem::path get_unresolved_path() const { return path; } + bool isRecursive() const { return !noRecurse; } + bool isImport() const { return import; } + bool exists() const; + InputFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = std::filesystem::current_path()); + +protected: + bool noRecurse = true; + std::filesystem::path path; + std::filesystem::path cwd; + bool resolved = false; + bool import = false; + virtual std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; + virtual std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; + bool dirContains(const std::filesystem::path& dir) const; +}; + +struct ImportFile : public InputFile { + ImportFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = std::filesystem::current_path()); + virtual std::filesystem::path resolved_relative() const override; + virtual std::filesystem::path resolved_absolute() const override; + virtual std::filesystem::path resolved_absolute_basedir() const override; +}; +} // namespace caprica \ No newline at end of file diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 5494100..971a4c7 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -89,7 +89,7 @@ static const std::unordered_set FAKE_SKYRIM_SCRIPTS_SET = { "fake://skyrim/DLC1SCWispWallScript.psc", }; -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, CapricaJobManager* jobManager, @@ -107,12 +107,12 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType time_t lastModTime, size_t fileSize); -bool addSingleFile(const conf::InputFile& input, +bool addSingleFile(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager *jobManager, PapyrusCompilationNode::NodeType nodeType); -bool addFilesFromDirectory(const conf::InputFile& input, +bool addFilesFromDirectory(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType, @@ -287,7 +287,7 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType return getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data.cFileName, lastModTime, fileSize); } -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { // Skyrim hacks; we need to import Skyrim's fake scripts into the global namespace first. if (conf::Papyrus::game == GameID::Skyrim) { caprica::caseless_unordered_identifier_ref_map tempMap{}; @@ -320,7 +320,7 @@ bool handleImports(const std::vector& f, caprica::CapricaJobMa return true; } -bool addSingleFile(const conf::InputFile& input, +bool addSingleFile(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType) { diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index cbc6bc2..bdadac1 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -21,14 +21,14 @@ namespace filesystem = std::filesystem; namespace caprica { struct CapricaJobManager; -bool addFilesFromDirectory(const conf::InputFile& input, +bool addFilesFromDirectory(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, papyrus::PapyrusCompilationNode::NodeType nodeType, const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); -bool addSingleFile(const conf::InputFile& input, +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); +bool addSingleFile(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType nodeType); From 77d6d9b320c6995edae166f8fcda0675d07a80af Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Wed, 11 Oct 2023 22:11:28 -0700 Subject: [PATCH 15/22] rename ImportFile to ImportDir --- Caprica/common/CapricaConfig.cpp | 2 +- Caprica/common/CapricaConfig.h | 2 +- Caprica/common/CapricaInputFile.cpp | 8 ++++---- Caprica/common/CapricaInputFile.h | 4 ++-- Caprica/main.cpp | 4 ++-- Caprica/main_options.cpp | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index 2e89f07..cd7ef77 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -59,7 +59,7 @@ GameID game { GameID::UNKNOWN }; bool enableLanguageExtensions{ false }; bool ignorePropertyNameLocalConflicts{ false }; bool allowImplicitNoneCastsToAnyType{ false }; - std::vector importDirectories {}; + std::vector importDirectories {}; CapricaUserFlagsDefinition userFlagsDefinition{}; } diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index 6ead0ef..cd1c6f6 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -113,7 +113,7 @@ namespace Papyrus { extern bool allowImplicitNoneCastsToAnyType; // The directories to search in for imported types and // unknown types. - extern std::vector importDirectories; + extern std::vector importDirectories; // The user flags definition. extern CapricaUserFlagsDefinition userFlagsDefinition; } diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index b1218bb..4563449 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -77,7 +77,7 @@ namespace caprica{ return std::filesystem::exists(resolved_absolute()); } - ImportFile::ImportFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + ImportDir::ImportDir(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) : InputFile(_path, noRecurse, _cwd) { import = true; // make the import path absolute @@ -86,15 +86,15 @@ namespace caprica{ path = FSUtils::canonicalFS(path); } - std::filesystem::path ImportFile::resolved_relative() const { + std::filesystem::path ImportDir::resolved_relative() const { return {}; } - std::filesystem::path ImportFile::resolved_absolute() const { + std::filesystem::path ImportDir::resolved_absolute() const { return path; // we always return the absolute path for imports } - std::filesystem::path ImportFile::resolved_absolute_basedir() const { + std::filesystem::path ImportDir::resolved_absolute_basedir() const { return path; } diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index 0506e42..9186e13 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -25,8 +25,8 @@ struct InputFile { bool dirContains(const std::filesystem::path& dir) const; }; -struct ImportFile : public InputFile { - ImportFile(const std::filesystem::path& _path, +struct ImportDir : public InputFile { + ImportDir(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = std::filesystem::current_path()); virtual std::filesystem::path resolved_relative() const override; diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 971a4c7..36d888a 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -89,7 +89,7 @@ static const std::unordered_set FAKE_SKYRIM_SCRIPTS_SET = { "fake://skyrim/DLC1SCWispWallScript.psc", }; -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, CapricaJobManager* jobManager, @@ -287,7 +287,7 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType return getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data.cFileName, lastModTime, fileSize); } -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { // Skyrim hacks; we need to import Skyrim's fake scripts into the global namespace first. if (conf::Papyrus::game == GameID::Skyrim) { caprica::caseless_unordered_identifier_ref_map tempMap{}; diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index bdadac1..01b48c8 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -27,7 +27,7 @@ bool addFilesFromDirectory(const InputFile& input, papyrus::PapyrusCompilationNode::NodeType nodeType, const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); -bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); +bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); bool addSingleFile(const InputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, From c6b872c0511733c5809e51cbecc7867e756fa9ef Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:13:10 -0700 Subject: [PATCH 16/22] Add back our current input resolution --- Caprica/common/CapricaConfig.cpp | 2 +- Caprica/common/CapricaConfig.h | 2 +- Caprica/common/CapricaInputFile.cpp | 201 +++++++++++++++++++++------- Caprica/common/CapricaInputFile.h | 75 ++++++++--- Caprica/common/FSUtils.h | 1 + Caprica/main.cpp | 6 +- Caprica/main_options.cpp | 41 +++--- 7 files changed, 235 insertions(+), 93 deletions(-) diff --git a/Caprica/common/CapricaConfig.cpp b/Caprica/common/CapricaConfig.cpp index cd7ef77..10896c6 100644 --- a/Caprica/common/CapricaConfig.cpp +++ b/Caprica/common/CapricaConfig.cpp @@ -13,7 +13,7 @@ namespace General { bool recursive { false }; std::filesystem::path outputDirectory; bool anonymizeOutput; - std::vector inputFiles; + std::vector> inputFiles; } namespace PCompiler { diff --git a/Caprica/common/CapricaConfig.h b/Caprica/common/CapricaConfig.h index cd1c6f6..4d07035 100644 --- a/Caprica/common/CapricaConfig.h +++ b/Caprica/common/CapricaConfig.h @@ -27,7 +27,7 @@ namespace General { // If true, remove identifying information from the header. extern bool anonymizeOutput; // input files - extern std::vector inputFiles; + extern std::vector> inputFiles; } // options related to compatibility with PCompiler's CLI parsing and name resolution diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index 4563449..398c410 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -3,50 +3,35 @@ #include #include #include +#include "CapricaInputFile.h" namespace caprica{ std::filesystem::path InputFile::resolved_relative() const { - for (auto& dir : conf::Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return get_relative_path(dir.resolved_absolute()); - return {}; // not found + if (!resolved) return {}; + return absPath.lexically_relative(absBaseDir); } std::filesystem::path InputFile::resolved_absolute() const { - // find the file among the import directories - for (auto& dir : conf::Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return get_absolute_path(dir.resolved_absolute()); - return {}; // not found + if (!resolved) return {}; + return absPath; } std::filesystem::path InputFile::resolved_absolute_basedir() const { - // find the file among the import directories - for (auto& dir : conf::Papyrus::importDirectories) - if (dirContains(dir.resolved_absolute())) - return dir.resolved_absolute(); - return {}; // not found - } - - std::filesystem::path InputFile::get_absolute_path(const std::filesystem::path& absDir) const { - if (path.is_absolute()) - return FSUtils::canonicalFS(path); - else - return FSUtils::canonicalFS((absDir / path)); + if (!resolved) return {}; + return absBaseDir; } - std::filesystem::path InputFile::get_relative_path(const std::filesystem::path& absDir) const { - std::filesystem::path cpath; - if (path.is_absolute()) - cpath = FSUtils::canonicalFS(path); - else - cpath = FSUtils::canonicalFS((absDir / path)); - return cpath.lexically_relative(absDir); + std::filesystem::path IInputFile::find_import_dir(const std::filesystem::path& path) { + for (auto& dir : conf::Papyrus::importDirectories) { + if (dirContains(path, dir.resolved_absolute())) { + return dir.resolved_absolute(); + } + } + return {}; } - bool InputFile::dirContains(const std::filesystem::path& dir) const { - + bool IInputFile::dirContains(const std::filesystem::path& path, const std::filesystem::path& dir) { if (path.is_absolute()) { // check if the path is contained in the import directory auto rel = path.lexically_relative(dir).string(); @@ -58,32 +43,57 @@ namespace caprica{ } return false; } - InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + + IInputFile::IInputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) : noRecurse(noRecurse), - path(std::move(_path)), - cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(_cwd)) { - path = path.make_preferred(); - // special handler; this points to something relative to the cwd, not an object path to be resolved - auto str = path.string(); - if (str == "." || str == ".." || str.starts_with(".\\") || str.starts_with("./") || str.contains("..\\") || - str.contains("../")) { - if (!path.is_absolute()) - path = cwd / path; - path = FSUtils::canonicalFS(path); - } + rawPath(std::move(_path)), + cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(FSUtils::canonicalFS(_cwd))) { } bool InputFile::exists() const { return std::filesystem::exists(resolved_absolute()); } + bool InputFile::isDir() const { + return std::filesystem::is_directory(resolved_absolute()); + } + + bool InputFile::resolve() { + auto normalPath = rawPath.lexically_normal(); + auto str = normalPath.string(); + if (!normalPath.is_absolute()) { + absPath = FSUtils::canonicalFS(cwd / normalPath); + absBaseDir = cwd; + if (std::filesystem::exists(absPath)) { + resolved = true; + return true; + } else { + return false; + } + } else { + absPath = FSUtils::canonicalFS(normalPath); + absBaseDir = find_import_dir(absPath); + if (absBaseDir.empty()) { + absBaseDir = absPath.parent_path(); + requiresPreParse = true; + } + } + + if (std::filesystem::exists(absPath)) { + resolved = true; + return true; + } + + return false; + } + + InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd): IInputFile(_path, noRecurse, _cwd) { + } + ImportDir::ImportDir(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : InputFile(_path, noRecurse, _cwd) { + : IInputFile(_path, noRecurse, _cwd) { import = true; - // make the import path absolute - if (!path.is_absolute()) - path = cwd / path; - path = FSUtils::canonicalFS(path); + resolve(); // we resolve import dirs immediately } std::filesystem::path ImportDir::resolved_relative() const { @@ -91,11 +101,102 @@ namespace caprica{ } std::filesystem::path ImportDir::resolved_absolute() const { - return path; // we always return the absolute path for imports + return resolvedPath; // we always return the absolute path for imports } std::filesystem::path ImportDir::resolved_absolute_basedir() const { - return path; + return resolvedPath; + } + + bool ImportDir::exists() const { + return std::filesystem::exists(resolvedPath); + } + + bool ImportDir::resolve() { + if (!rawPath.is_absolute()) + resolvedPath = FSUtils::canonicalFS(cwd / rawPath); + else + resolvedPath = FSUtils::canonicalFS(rawPath); + + if (std::filesystem::exists(resolvedPath) && std::filesystem::is_directory(resolvedPath)) { + resolved = true; + return true; + } + return false; + } + + PCompInputFile::PCompInputFile(const std::filesystem::path& _path, + bool noRecurse, + bool isFolder, + const std::filesystem::path& _cwd): IInputFile(_path, noRecurse, _cwd) { + __isFolder = isFolder; + } + + std::filesystem::path PCompInputFile::resolved_relative() const { + if (!resolved) return {}; + return absPath.lexically_relative(absBaseDir); + } + + std::filesystem::path PCompInputFile::resolved_absolute() const { + if (!resolved) return {}; + return absPath; + } + + std::filesystem::path PCompInputFile::resolved_absolute_basedir() const { + if (!resolved) return {}; + return absBaseDir; + } + + bool PCompInputFile::exists() const { + return std::filesystem::exists(resolved_absolute()); + } + + bool PCompInputFile::resolve() { + auto normalPath = rawPath; + auto str = normalPath.string(); + // replace all ':' with preferred seperator + std::replace(str.begin(), str.end(), ':', FSUtils::SEP); + normalPath = str; + if (!__isFolder && normalPath.extension().empty()) + normalPath.replace_extension(".psc"); + normalPath = normalPath.lexically_normal(); + std::string sep = {FSUtils::SEP}; + str = normalPath.string(); + + + // special case for relative paths that contain parent/cwd refs + if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with("." + sep) || str.contains(".." + sep))) { + absPath = FSUtils::canonicalFS(cwd / normalPath); + absBaseDir = cwd; + + if (!std::filesystem::exists(absPath)) + return false; + resolved = true; + return true; + } + + // if this is a relative folder path, and the folder is in the cwd, use cwd as the base dir + if (__isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) { + absBaseDir = cwd; + } else { + absBaseDir = find_import_dir(normalPath); + } + + if (absBaseDir.empty()) { + return false; + } + + if (!normalPath.is_absolute()) { + absPath = FSUtils::canonicalFS(absBaseDir / normalPath); + } else { + absPath = FSUtils::canonicalFS(normalPath); + } + + if (std::filesystem::exists(absPath)) { + resolved = true; + return true; + } + return false; } -} // namespace caprica \ No newline at end of file +} // namespace caprica diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index 9186e13..ef47235 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -1,36 +1,79 @@ #pragma once #include namespace caprica{ - -struct InputFile { - virtual std::filesystem::path resolved_relative() const; - virtual std::filesystem::path resolved_absolute() const; - virtual std::filesystem::path resolved_absolute_basedir() const; - std::filesystem::path get_unresolved_path() const { return path; } +struct IInputFile; +struct IInputFile : std::enable_shared_from_this{ + virtual std::filesystem::path resolved_relative() const = 0; + virtual std::filesystem::path resolved_absolute() const = 0; + virtual std::filesystem::path resolved_absolute_basedir() const = 0; + std::filesystem::path get_unresolved_path() const { return rawPath; } bool isRecursive() const { return !noRecurse; } bool isImport() const { return import; } - bool exists() const; - InputFile(const std::filesystem::path& _path, + virtual bool exists() const = 0; + virtual bool isDir() const = 0; + virtual bool resolve() = 0; + virtual bool isResolved() const { return resolved; } + IInputFile(const std::filesystem::path& _path, bool noRecurse = true, - const std::filesystem::path& _cwd = std::filesystem::current_path()); + const std::filesystem::path& _cwd = ""); + virtual ~IInputFile() = default; + // static InputFile createNewStyle(const std::filesystem::path& path, bool noRecurse = true, const std::filesystem::path& cwd = std::filesystem::current_path()); + // static InputFile createOldStyle(const std::filesystem::path& path, bool noRecurse = true, const std::filesystem::path& cwd = std::filesystem::current_path(), bool isFolder = false); protected: bool noRecurse = true; - std::filesystem::path path; - std::filesystem::path cwd; + const std::filesystem::path rawPath; + const std::filesystem::path cwd; bool resolved = false; bool import = false; - virtual std::filesystem::path get_absolute_path(const std::filesystem::path& absDir) const; - virtual std::filesystem::path get_relative_path(const std::filesystem::path& dir) const; - bool dirContains(const std::filesystem::path& dir) const; + bool requiresPreParse = false; + static std::filesystem::path find_import_dir(const std::filesystem::path& _path); + static bool dirContains(const std::filesystem::path& _path, const std::filesystem::path& dir); +}; +struct InputFile : public IInputFile { + virtual std::filesystem::path resolved_relative() const override; + virtual std::filesystem::path resolved_absolute() const override; + virtual std::filesystem::path resolved_absolute_basedir() const override; + virtual bool exists() const override; + virtual bool isDir() const override; + virtual bool resolve() override; + InputFile(const std::filesystem::path& _path, + bool noRecurse = true, + const std::filesystem::path& _cwd = ""); + virtual ~InputFile() = default; + + private: + std::filesystem::path absPath; + std::filesystem::path absBaseDir; +}; + +struct PCompInputFile : public IInputFile { + PCompInputFile(const std::filesystem::path& _path, + bool noRecurse = true, bool isFolder = false, + const std::filesystem::path& _cwd = ""); + virtual std::filesystem::path resolved_relative() const override; + virtual std::filesystem::path resolved_absolute() const override; + virtual std::filesystem::path resolved_absolute_basedir() const override; + virtual bool exists() const override; + virtual bool isDir() const override { return __isFolder; } + virtual bool resolve() override; + private: + bool __isFolder = false; + std::filesystem::path absPath; + std::filesystem::path absBaseDir; }; -struct ImportDir : public InputFile { +struct ImportDir : public IInputFile { ImportDir(const std::filesystem::path& _path, bool noRecurse = true, - const std::filesystem::path& _cwd = std::filesystem::current_path()); + const std::filesystem::path& _cwd = ""); virtual std::filesystem::path resolved_relative() const override; virtual std::filesystem::path resolved_absolute() const override; virtual std::filesystem::path resolved_absolute_basedir() const override; + virtual bool exists() const override; + virtual bool isDir() const override { return true; } + virtual bool resolve() override; + private: + std::filesystem::path resolvedPath; }; } // namespace caprica \ No newline at end of file diff --git a/Caprica/common/FSUtils.h b/Caprica/common/FSUtils.h index b4849d7..a141051 100644 --- a/Caprica/common/FSUtils.h +++ b/Caprica/common/FSUtils.h @@ -9,6 +9,7 @@ namespace caprica { namespace FSUtils { +static constexpr char SEP = std::filesystem::path::preferred_separator; std::string_view basenameAsRef(std::string_view file); std::string_view extensionAsRef(std::string_view file); std::string_view filenameAsRef(std::string_view file); diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 36d888a..e6f9d14 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -107,12 +107,12 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType time_t lastModTime, size_t fileSize); -bool addSingleFile(const InputFile& input, +bool addSingleFile(const IInputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager *jobManager, PapyrusCompilationNode::NodeType nodeType); -bool addFilesFromDirectory(const InputFile& input, +bool addFilesFromDirectory(const IInputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType, @@ -320,7 +320,7 @@ bool handleImports(const std::vector& f, caprica::CapricaJobManager* return true; } -bool addSingleFile(const InputFile& input, +bool addSingleFile(const IInputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, PapyrusCompilationNode::NodeType nodeType) { diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index 01b48c8..c4f5a46 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -21,14 +21,14 @@ namespace filesystem = std::filesystem; namespace caprica { struct CapricaJobManager; -bool addFilesFromDirectory(const InputFile& input, +bool addFilesFromDirectory(const IInputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, papyrus::PapyrusCompilationNode::NodeType nodeType, const std::string& startingNS = ""); void parseUserFlags(std::string&& flagsPath); bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager); -bool addSingleFile(const InputFile& input, +bool addSingleFile(const IInputFile& input, const std::filesystem::path& baseOutputDir, caprica::CapricaJobManager* jobManager, caprica::papyrus::PapyrusCompilationNode::NodeType nodeType); @@ -468,10 +468,11 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage baseOutputDir = outputDir; conf::General::inputFiles.reserve(conf::General::inputFiles.size() + ppj.folders.size() + ppj.scripts.size()); - for (auto& f : ppj.folders) - conf::General::inputFiles.emplace_back(f.path, f.noRecurse, baseDir); + for (auto& f : ppj.folders){ + conf::General::inputFiles.emplace_back(std::make_shared(f.path, f.noRecurse, true, baseDir)); + } for (auto& f : ppj.scripts) - conf::General::inputFiles.emplace_back(f, false, baseDir); + conf::General::inputFiles.emplace_back(std::make_shared(f, true, false, baseDir)); flags = ppj.flags; } else { // if we're not doing a project file... // ensure cwd is placed first @@ -641,6 +642,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage // PCompiler input resolution if (conf::PCompiler::pCompilerCompatibilityMode) { for (auto& f : filesPassed) { + bool isFolder = conf::PCompiler::all; if (conf::PCompiler::all) { if (!std::filesystem::is_directory(f)) { std::cout << "Unable to locate input directory '" << f << "'." << std::endl; @@ -652,11 +654,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage if (FSUtils::extensionAsRef(f).empty()) f.append(".psc"); } - auto input = conf::General::inputFiles.emplace_back(f, !iterateCompiledDirectoriesRecursively); - if (!std::filesystem::exists(input.resolved_absolute())) { - std::cout << "Unable to locate input file '" << f << "'." << std::endl; - return false; - } + auto & input = conf::General::inputFiles.emplace_back(std::make_shared(f, !iterateCompiledDirectoriesRecursively, isFolder)); } } else { // normal resolution @@ -670,33 +668,32 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage return false; } } - auto& input = conf::General::inputFiles.emplace_back(f, !iterateCompiledDirectoriesRecursively); - if (!std::filesystem::exists(input.resolved_absolute())) { - std::cout << "Unable to locate input file '" << f << "'." << std::endl; - return false; - } + auto& input = conf::General::inputFiles.emplace_back(std::make_shared(f, !iterateCompiledDirectoriesRecursively)); } } for (auto& input : conf::General::inputFiles) { - if (!input.exists()) { - std::cout << "Unable to locate input file '" << input.get_unresolved_path() << "'." << std::endl; + if (!input){ + throw std::runtime_error("Input file is null!"); + } + if (!input->resolve()) { + std::cout << "Unable to locate input file '" << input->get_unresolved_path() << "'." << std::endl; return false; } - if (std::filesystem::is_directory(input.resolved_absolute())) { - if (!addFilesFromDirectory(input, + if (std::filesystem::is_directory(input->resolved_absolute())) { + if (!addFilesFromDirectory(*input, baseOutputDir, jobManager, papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile, "")) { - std::cout << "Unable to add input directory '" << input.get_unresolved_path() << "'." << std::endl; + std::cout << "Unable to add input directory '" << input->get_unresolved_path() << "'." << std::endl; return false; } - } else if (!addSingleFile(input, + } else if (!addSingleFile(*input, baseOutputDir, jobManager, papyrus::PapyrusCompilationNode::NodeType::PapyrusCompile)) { - std::cout << "Unable to add input file '" << input.get_unresolved_path() << "'." << std::endl; + std::cout << "Unable to add input file '" << input->get_unresolved_path() << "'." << std::endl; return false; } } From 8181e1e5d3d45996b8ceb72981e3d8a32d3543a8 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:38:14 -0700 Subject: [PATCH 17/22] add object name converters, replace '\\' with FSUtils::SEP os independence! --- Caprica/common/CapricaInputFile.cpp | 20 ++--- Caprica/common/CapricaInputFile.h | 3 +- Caprica/common/FSUtils.cpp | 74 +++++++++++++------ Caprica/common/FSUtils.h | 5 +- Caprica/main.cpp | 23 +++--- Caprica/main_options.cpp | 6 +- Caprica/papyrus/PapyrusCompilationContext.cpp | 19 ++--- 7 files changed, 85 insertions(+), 65 deletions(-) diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index 398c410..8e264b2 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -59,7 +59,7 @@ namespace caprica{ } bool InputFile::resolve() { - auto normalPath = rawPath.lexically_normal(); + auto normalPath = FSUtils::normalize(rawPath); auto str = normalPath.string(); if (!normalPath.is_absolute()) { absPath = FSUtils::canonicalFS(cwd / normalPath); @@ -150,22 +150,18 @@ namespace caprica{ bool PCompInputFile::exists() const { return std::filesystem::exists(resolved_absolute()); } + + static constexpr char const curDir[3] = {'.', FSUtils::SEP, 0}; + static constexpr char const parent[4] = {'.', '.', FSUtils::SEP, 0}; bool PCompInputFile::resolve() { - auto normalPath = rawPath; - auto str = normalPath.string(); - // replace all ':' with preferred seperator - std::replace(str.begin(), str.end(), ':', FSUtils::SEP); - normalPath = str; + std::filesystem::path normalPath = FSUtils::objectNameToPath(rawPath.string()); if (!__isFolder && normalPath.extension().empty()) normalPath.replace_extension(".psc"); - normalPath = normalPath.lexically_normal(); - std::string sep = {FSUtils::SEP}; - str = normalPath.string(); - - + normalPath = FSUtils::normalize(normalPath); + std::string str = normalPath.string(); // special case for relative paths that contain parent/cwd refs - if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with("." + sep) || str.contains(".." + sep))) { + if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with(curDir) || str.contains(parent))) { absPath = FSUtils::canonicalFS(cwd / normalPath); absBaseDir = cwd; diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index ef47235..665a3e5 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -2,7 +2,7 @@ #include namespace caprica{ struct IInputFile; -struct IInputFile : std::enable_shared_from_this{ +struct IInputFile { virtual std::filesystem::path resolved_relative() const = 0; virtual std::filesystem::path resolved_absolute() const = 0; virtual std::filesystem::path resolved_absolute_basedir() const = 0; @@ -30,6 +30,7 @@ struct IInputFile : std::enable_shared_from_this{ static std::filesystem::path find_import_dir(const std::filesystem::path& _path); static bool dirContains(const std::filesystem::path& _path, const std::filesystem::path& dir); }; + struct InputFile : public IInputFile { virtual std::filesystem::path resolved_relative() const override; virtual std::filesystem::path resolved_absolute() const override; diff --git a/Caprica/common/FSUtils.cpp b/Caprica/common/FSUtils.cpp index b1a0884..8af7411 100644 --- a/Caprica/common/FSUtils.cpp +++ b/Caprica/common/FSUtils.cpp @@ -46,11 +46,13 @@ std::string_view parentPathAsRef(std::string_view file) { return file; } +static constexpr char Redundancy[3] = {FSUtils::SEP, FSUtils::SEP, '\0'}; +static constexpr char CWDRef[4] = {FSUtils::SEP, '.', FSUtils::SEP, '\0'}; bool shouldShortCircuit(const std::string& path) { if (path.size() > 3 && path[1] == ':') { // Shortcircuit for already canon paths. if (path.find("/") == std::string::npos && path.find("..") == std::string::npos && - path.find("\\.\\") == std::string::npos && path.find("\\\\") == std::string::npos) { + path.find(CWDRef) == std::string::npos && path.find(Redundancy) == std::string::npos) { return true; } } @@ -58,7 +60,55 @@ bool shouldShortCircuit(const std::string& path) { } // Borrowed and modified from http://stackoverflow.com/a/1750710/776797 +std::filesystem::path normalize(const std::filesystem::path& path) { +#ifdef _WIN32 + std::filesystem::path result; + for (auto it = path.begin(); it != path.end(); ++it) { + if (!wcscmp(it->c_str(), L"..")) { + // /a/b/../.. is not /a/b/.. under most circumstances + // We can end up with ..s in our result because of symbolic links + if (!wcscmp(result.filename().c_str(), L"..")) + result /= *it; + // Otherwise it should be safe to resolve the parent + else + result = result.parent_path(); + } else if (!wcscmp(it->c_str(), L".")) { + // Ignore + } else { + // Just cat other path entries + result /= *it; + } + } + return result.make_preferred(); +#else + return path.lexically_normal(); +#endif +} + +std::string pathToObjectName(const std::filesystem::path& path) { + if (path.empty()) return ""; + std::string result = normalize(path).string(); + std::replace(result.begin(), result.end(), FSUtils::SEP, ':'); + auto pos = result.rfind('.'); + if (pos != std::string_view::npos) + return result.substr(0, pos); + return result; +} + +std::filesystem::path objectNameToPath(const std::string& objectName) { + if (objectName.empty()) return ""; + std::string result; + std::transform(objectName.begin(), objectName.end(), std::back_inserter(result), [](char c) { + if (c == ':') + return FSUtils::SEP; + return c; + }); + return result; +} + std::string canonical(const std::string& path) { + if (shouldShortCircuit(path)) + return path; return canonicalFS(path).string(); } @@ -67,27 +117,9 @@ std::filesystem::path canonicalFS(const std::filesystem::path& path) { return path; auto absPath = std::filesystem::absolute(path); if (conf::Performance::resolveSymlinks) { - return std::filesystem::canonical(absPath).make_preferred().string(); - } else { - std::filesystem::path result; - for (auto it = absPath.begin(); it != absPath.end(); ++it) { - if (!wcscmp(it->c_str(), L"..")) { - // /a/b/../.. is not /a/b/.. under most circumstances - // We can end up with ..s in our result because of symbolic links - if (!wcscmp(result.filename().c_str(), L"..")) - result /= *it; - // Otherwise it should be safe to resolve the parent - else - result = result.parent_path(); - } else if (!wcscmp(it->c_str(), L".")) { - // Ignore - } else { - // Just cat other path entries - result /= *it; - } - } - return result.make_preferred(); + return std::filesystem::canonical(absPath).make_preferred(); } + return normalize(absPath); } }} diff --git a/Caprica/common/FSUtils.h b/Caprica/common/FSUtils.h index a141051..5292c5e 100644 --- a/Caprica/common/FSUtils.h +++ b/Caprica/common/FSUtils.h @@ -9,12 +9,15 @@ namespace caprica { namespace FSUtils { -static constexpr char SEP = std::filesystem::path::preferred_separator; +static constexpr inline char SEP = std::filesystem::path::preferred_separator; std::string_view basenameAsRef(std::string_view file); std::string_view extensionAsRef(std::string_view file); std::string_view filenameAsRef(std::string_view file); std::string_view parentPathAsRef(std::string_view file); +std::filesystem::path objectNameToPath(const std::string& objectName); +std::string pathToObjectName(const std::filesystem::path& path); +std::filesystem::path normalize(const std::filesystem::path& path); std::string canonical(const std::string& path); std::filesystem::path canonicalFS(const std::filesystem::path& path); diff --git a/Caprica/main.cpp b/Caprica/main.cpp index e6f9d14..8769dff 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -216,11 +216,8 @@ bool addFilesFromDirectory(const IInputFile& input, l_startNS = ""; } } - // if it's true, then this is the base dir and the namespace should be root - auto namespaceName = l_startNS + curDir.lexically_normal().string(); - std::replace(namespaceName.begin(), namespaceName.end(), '\\', ':'); - while (namespaceName[0] == ':') - namespaceName = namespaceName.substr(1); + auto curDirNS = FSUtils::pathToObjectName(curDir); + auto namespaceName = l_startNS.empty() ? curDirNS : (l_startNS + ":" + curDirNS); caprica::papyrus::PapyrusCompilationContext::pushNamespaceFullContents(namespaceName, std::move(namespaceMap)); } } else { @@ -327,11 +324,11 @@ bool addSingleFile(const IInputFile& input, // Get the file size and last modified time using std::filesystem std::error_code ec; - auto relPath = input.resolved_relative(); - auto namespaceDir = relPath.parent_path(); - auto absPath = input.resolved_absolute(); - auto filename = absPath.filename(); - auto absBaseDir = input.resolved_absolute_basedir(); + const auto relPath = input.resolved_relative(); + const auto namespaceDir = relPath.parent_path(); + const auto absPath = input.resolved_absolute(); + const auto filename = absPath.filename(); + const auto absBaseDir = input.resolved_absolute_basedir(); if (!input.exists()) { std::cout << "ERROR: Resolved file path '" << absPath << "' is not in an import directory, cannot resolve namespace!" << std::endl; @@ -347,10 +344,8 @@ bool addSingleFile(const IInputFile& input, std::cout << "An error occurred while trying to get the file size of '" << absPath << "'!" << std::endl; return false; } - auto namespaceName = relPath.parent_path().string(); - std::replace(namespaceName.begin(), namespaceName.end(), '\\', ':'); - if (namespaceName[0] == ':') - namespaceName = namespaceName.substr(1); + + auto namespaceName = FSUtils::pathToObjectName(namespaceDir); if (!conf::General::quietCompile) std::cout << "Adding file '" << filename << "' to namespace '" << namespaceName << "'." << std::endl; auto node = getNode(nodeType, diff --git a/Caprica/main_options.cpp b/Caprica/main_options.cpp index c4f5a46..5347c94 100644 --- a/Caprica/main_options.cpp +++ b/Caprica/main_options.cpp @@ -381,7 +381,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage std::filesystem::path baseOutputDir; PapyrusProject ppj; - std::string ppjPath; + std::filesystem::path ppjPath; auto filesToRemove = std::vector(); for (auto& f : filesPassed) { auto ext = filesystem::path(f).extension().string(); @@ -430,7 +430,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } if (!ppjPath.empty()) { - auto baseDir = filesystem::path(ppjPath).parent_path(); + auto baseDir = ppjPath.parent_path(); if (!baseDir.empty()) baseDir = FSUtils::canonicalFS(baseDir); else @@ -650,7 +650,7 @@ bool parseCommandLineArguments(int argc, char* argv[], caprica::CapricaJobManage } } else { // need to replace any `:` with `\` - std::replace(f.begin(), f.end(), ':', '\\'); + std::replace(f.begin(), f.end(), ':', FSUtils::SEP); if (FSUtils::extensionAsRef(f).empty()) f.append(".psc"); } diff --git a/Caprica/papyrus/PapyrusCompilationContext.cpp b/Caprica/papyrus/PapyrusCompilationContext.cpp index dccfea1..7c49f8c 100644 --- a/Caprica/papyrus/PapyrusCompilationContext.cpp +++ b/Caprica/papyrus/PapyrusCompilationContext.cpp @@ -164,20 +164,13 @@ void PapyrusCompilationNode::FilePreParseJob::run() { case NodeType::PapyrusCompile: // only check for this on compile nodes case NodeType::PasCompile: { // check the object name with the reportedname - auto nsName = std::string(FSUtils::parentPathAsRef(parent->reportedName)); - if (nsName == parent->reportedName) - nsName = ""; - // replace `\` with `:` - std::replace(nsName.begin(), nsName.end(), '\\', ':'); - auto objectNS = parent->objectName.find(':') != std::string::npos - ? parent->objectName.substr(0, parent->objectName.find_last_of(':')) - : ""; - if (_strnicmp(objectNS.c_str(), nsName.c_str(), objectNS.size()) != 0) { + auto nsName = FSUtils::pathToObjectName(parent->reportedName); + if (_strnicmp(parent->objectName.c_str(), nsName.c_str(), parent->objectName.size()) != 0) { CapricaReportingContext::logicalFatal( "{}: The script namespace '{}' does not match the expected namespace '{}'.\n" "Check your imports and your CWD.", parent->reportedName, - objectNS, + parent->objectName, nsName); } } break; @@ -269,7 +262,7 @@ void PapyrusCompilationNode::FileCompileJob::run() { auto containingDir = std::filesystem::path(parent->outputDirectory); if (!std::filesystem::exists(containingDir)) std::filesystem::create_directories(containingDir); - std::ofstream asmStrm(parent->outputDirectory + "\\" + std::string(parent->baseName) + ".pas", + std::ofstream asmStrm(parent->outputDirectory + FSUtils::SEP + std::string(parent->baseName) + ".pas", std::ofstream::binary); asmStrm.exceptions(std::ifstream::badbit | std::ifstream::failbit); pex::PexAsmWriter asmWtr(asmStrm); @@ -286,7 +279,7 @@ void PapyrusCompilationNode::FileCompileJob::run() { auto containingDir = std::filesystem::path(parent->outputDirectory); if (!std::filesystem::exists(containingDir)) std::filesystem::create_directories(containingDir); - std::ofstream asmStrm(parent->outputDirectory + "\\" + std::string(parent->baseName) + ".pas", + std::ofstream asmStrm(parent->outputDirectory + FSUtils::SEP + std::string(parent->baseName) + ".pas", std::ofstream::binary); asmStrm.exceptions(std::ifstream::badbit | std::ifstream::failbit); caprica::pex::PexAsmWriter asmWtr(asmStrm); @@ -328,7 +321,7 @@ void PapyrusCompilationNode::FileWriteJob::run() { auto containingDir = std::filesystem::path(parent->outputDirectory); if (!std::filesystem::exists(containingDir)) std::filesystem::create_directories(containingDir); - std::ofstream destFile { parent->outputDirectory + "\\" + baseFileName + ".pex", std::ifstream::binary }; + std::ofstream destFile { parent->outputDirectory + FSUtils::SEP + baseFileName + ".pex", std::ifstream::binary }; destFile.exceptions(std::ifstream::badbit | std::ifstream::failbit); parent->pexWriter->applyToBuffers([&](const char* data, size_t size) { destFile.write(data, size); }); } From e58a3e1e802dbc68b383cf273d7a69bf74859cf3 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 18:03:49 -0700 Subject: [PATCH 18/22] fix bug in script name finder --- Caprica/papyrus/PapyrusCompilationContext.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Caprica/papyrus/PapyrusCompilationContext.cpp b/Caprica/papyrus/PapyrusCompilationContext.cpp index 7c49f8c..e2c43db 100644 --- a/Caprica/papyrus/PapyrusCompilationContext.cpp +++ b/Caprica/papyrus/PapyrusCompilationContext.cpp @@ -128,7 +128,7 @@ std::string_view findScriptName(const std::string_view& data, const std::string_ auto line = data.substr(last, next - last); auto begin = line.find_first_not_of(" \t"); if (strnicmp(line.substr(begin, startstring.size()).data(), startstring.data(), startstring.size()) == 0) { - auto first = line.find_first_not_of(" \t", startstring.size()); + auto first = line.find_first_not_of(" \t", startstring.size() + begin); return line.substr(first, line.find_first_of(" \t", first) - first); } last = next + 1; From ee8cebbf70f20bdc4de84d7c63bd7c1faf19c394 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 19:20:11 -0700 Subject: [PATCH 19/22] cleanup caprica input file --- .github/workflows/build.yml | 29 ++- Caprica/common/CapricaInputFile.cpp | 325 +++++++++++++--------------- Caprica/common/CapricaInputFile.h | 72 +++--- 3 files changed, 208 insertions(+), 218 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1b3379..4f6b546 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,10 +5,13 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + release: + types: [created] env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release + PROJECT_NAME: Caprica CMAKE_ARGS: "-DCHAMPOLLION_USE_STATIC_RUNTIME:BOOL=TRUE -DENABLE_STATIC_RUNTIME:BOOL=TRUE -DCMAKE_INSTALL_PREFIX:STRING=build/extern -DVCPKG_TARGET_TRIPLET:STRING=x64-windows-static -DCMAKE_TOOLCHAIN_FILE:STRING=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" jobs: @@ -51,8 +54,32 @@ jobs: uses: actions/upload-artifact@v3.1.2 with: # Artifact name - name: Caprica + name: ${{ env.PROJECT_NAME }} # A file, directory or wildcard pattern that describes what to upload path: build/Caprica/Release/Caprica.exe # The desired behavior if no files are found using the provided path. retention-days: 90 + + + release: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + contents: write + needs: build + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ env.PROJECT_NAME }} + path: artifacts/${{ env.PROJECT_NAME }} + - name: Zip artifacts + run: | + ls -la artifacts/* + cd artifacts/${{ env.PROJECT_NAME }} + zip -r9 "../${{ env.PROJECT_NAME }}-${{ github.ref_name }}.zip" * + - name: Release + uses: nikitalita/action-gh-release@v1.0 + with: + files: | + artifacts/${{ env.PROJECT_NAME }}-${{ github.ref_name }}.zip diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index 8e264b2..68cbe76 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -1,198 +1,181 @@ -#include +#include "CapricaInputFile.h" +#include #include #include #include -#include -#include "CapricaInputFile.h" - -namespace caprica{ +#include - std::filesystem::path InputFile::resolved_relative() const { - if (!resolved) return {}; - return absPath.lexically_relative(absBaseDir); - } +namespace caprica { - std::filesystem::path InputFile::resolved_absolute() const { - if (!resolved) return {}; - return absPath; - } - - std::filesystem::path InputFile::resolved_absolute_basedir() const { - if (!resolved) return {}; - return absBaseDir; - } - - std::filesystem::path IInputFile::find_import_dir(const std::filesystem::path& path) { - for (auto& dir : conf::Papyrus::importDirectories) { - if (dirContains(path, dir.resolved_absolute())) { - return dir.resolved_absolute(); - } - } +std::filesystem::path IInputFile::resolved_relative() const { + if (!resolved) return {}; - } - - bool IInputFile::dirContains(const std::filesystem::path& path, const std::filesystem::path& dir) { - if (path.is_absolute()) { - // check if the path is contained in the import directory - auto rel = path.lexically_relative(dir).string(); - if (!rel.empty() && !rel.starts_with("..")) - return true; - } else { - if (std::filesystem::exists(dir / path)) - return true; - } - return false; - } + auto rel = absPath.lexically_relative(absBaseDir); + if (rel == ".") + return {}; + return rel; +} - IInputFile::IInputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : noRecurse(noRecurse), - rawPath(std::move(_path)), - cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(FSUtils::canonicalFS(_cwd))) { - } +std::filesystem::path IInputFile::resolved_absolute() const { + if (!resolved) + return {}; + return absPath; +} - bool InputFile::exists() const { - return std::filesystem::exists(resolved_absolute()); - } +std::filesystem::path IInputFile::resolved_absolute_basedir() const { + if (!resolved) + return {}; + return absBaseDir; +} - bool InputFile::isDir() const { - return std::filesystem::is_directory(resolved_absolute()); +std::filesystem::path IInputFile::resolved_relative_parent() const { + if (!resolved) + return {}; + auto rel = resolved_relative(); + if (rel.empty()) + return {}; + return rel.parent_path(); +} + +std::filesystem::path IInputFile::find_import_dir(const std::filesystem::path& path) { + for (auto& dir : conf::Papyrus::importDirectories) + if (dirContains(path, dir.resolved_absolute())) + return dir.resolved_absolute(); + return {}; +} + +bool IInputFile::dirContains(const std::filesystem::path& path, const std::filesystem::path& dir) { + if (path.is_absolute()) { + // check if the path is contained in the import directory + auto rel = path.lexically_relative(dir).string(); + if (!rel.empty() && !rel.starts_with("..")) + return true; + } else { + if (std::filesystem::exists(dir / path)) + return true; } + return false; +} - bool InputFile::resolve() { - auto normalPath = FSUtils::normalize(rawPath); - auto str = normalPath.string(); - if (!normalPath.is_absolute()) { - absPath = FSUtils::canonicalFS(cwd / normalPath); - absBaseDir = cwd; - if (std::filesystem::exists(absPath)) { - resolved = true; - return true; - } else { - return false; - } - } else { - absPath = FSUtils::canonicalFS(normalPath); - absBaseDir = find_import_dir(absPath); - if (absBaseDir.empty()) { - absBaseDir = absPath.parent_path(); - requiresPreParse = true; - } - } +IInputFile::IInputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : noRecurse(noRecurse), + rawPath(std::move(_path)), + cwd(_cwd.empty() ? std::filesystem::current_path() : std::move(FSUtils::canonicalFS(_cwd))) { +} +bool IInputFile::exists() const { + if (!resolved) + return false; + return std::filesystem::exists(resolved_absolute()); +} + +bool IInputFile::isDir() const { + return std::filesystem::is_directory(resolved_absolute()); +} + +bool InputFile::resolve() { + auto normalPath = FSUtils::normalize(rawPath); + auto str = normalPath.string(); + if (!normalPath.is_absolute()) { + absPath = FSUtils::canonicalFS(cwd / normalPath); + absBaseDir = cwd; if (std::filesystem::exists(absPath)) { resolved = true; return true; + } else { + return false; } - - return false; - } - - InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd): IInputFile(_path, noRecurse, _cwd) { - } - - ImportDir::ImportDir(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : IInputFile(_path, noRecurse, _cwd) { - import = true; - resolve(); // we resolve import dirs immediately - } - - std::filesystem::path ImportDir::resolved_relative() const { - return {}; - } - - std::filesystem::path ImportDir::resolved_absolute() const { - return resolvedPath; // we always return the absolute path for imports - } - - std::filesystem::path ImportDir::resolved_absolute_basedir() const { - return resolvedPath; - } - - bool ImportDir::exists() const { - return std::filesystem::exists(resolvedPath); - } - - bool ImportDir::resolve() { - if (!rawPath.is_absolute()) - resolvedPath = FSUtils::canonicalFS(cwd / rawPath); - else - resolvedPath = FSUtils::canonicalFS(rawPath); - - if (std::filesystem::exists(resolvedPath) && std::filesystem::is_directory(resolvedPath)) { - resolved = true; - return true; + } else { + absPath = FSUtils::canonicalFS(normalPath); + absBaseDir = find_import_dir(absPath); + if (absBaseDir.empty()) { + absBaseDir = absPath.parent_path(); + requiresPreParse = true; } - return false; - } - - PCompInputFile::PCompInputFile(const std::filesystem::path& _path, - bool noRecurse, - bool isFolder, - const std::filesystem::path& _cwd): IInputFile(_path, noRecurse, _cwd) { - __isFolder = isFolder; - } - - std::filesystem::path PCompInputFile::resolved_relative() const { - if (!resolved) return {}; - return absPath.lexically_relative(absBaseDir); - } - - std::filesystem::path PCompInputFile::resolved_absolute() const { - if (!resolved) return {}; - return absPath; } - std::filesystem::path PCompInputFile::resolved_absolute_basedir() const { - if (!resolved) return {}; - return absBaseDir; + if (std::filesystem::exists(absPath)) { + resolved = true; + return true; + } + + return false; +} + +InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : IInputFile(_path, noRecurse, _cwd) { + requiresPreParse = true; // we always require pre-parse for non-PCompiler-compatible input files +} + +ImportDir::ImportDir(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : IInputFile(_path, noRecurse, _cwd) { + requiresPreParse = true; // we always require pre-parse for import dirs + import = true; + resolve(); // we resolve import dirs immediately +} + +bool ImportDir::resolve() { + if (!rawPath.is_absolute()) + absPath = FSUtils::canonicalFS(cwd / rawPath); + else + absPath = FSUtils::canonicalFS(rawPath); + absBaseDir = absPath; + if (std::filesystem::exists(absPath) && std::filesystem::is_directory(absPath)) { + resolved = true; + return true; + } + return false; +} + +PCompInputFile::PCompInputFile(const std::filesystem::path& _path, + bool noRecurse, + bool isFolder, + const std::filesystem::path& _cwd) + : IInputFile(_path, noRecurse, _cwd) { + __isFolder = isFolder; +} + +static constexpr char const curDir[3] = { '.', FSUtils::SEP, 0 }; +static constexpr char const parent[4] = { '.', '.', FSUtils::SEP, 0 }; + +bool PCompInputFile::resolve() { + std::filesystem::path normalPath = FSUtils::objectNameToPath(rawPath.string()); + if (!__isFolder && normalPath.extension().empty()) + normalPath.replace_extension(".psc"); + normalPath = FSUtils::normalize(normalPath); + std::string str = normalPath.string(); + // special case for relative paths that contain parent/cwd refs + if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with(curDir) || str.contains(parent))) { + absPath = FSUtils::canonicalFS(cwd / normalPath); + absBaseDir = cwd; + + if (!std::filesystem::exists(absPath)) + return false; + resolved = true; + return true; } - bool PCompInputFile::exists() const { - return std::filesystem::exists(resolved_absolute()); - } - - static constexpr char const curDir[3] = {'.', FSUtils::SEP, 0}; - static constexpr char const parent[4] = {'.', '.', FSUtils::SEP, 0}; - - bool PCompInputFile::resolve() { - std::filesystem::path normalPath = FSUtils::objectNameToPath(rawPath.string()); - if (!__isFolder && normalPath.extension().empty()) - normalPath.replace_extension(".psc"); - normalPath = FSUtils::normalize(normalPath); - std::string str = normalPath.string(); - // special case for relative paths that contain parent/cwd refs - if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with(curDir) || str.contains(parent))) { - absPath = FSUtils::canonicalFS(cwd / normalPath); - absBaseDir = cwd; - - if (!std::filesystem::exists(absPath)) - return false; - resolved = true; - return true; - } + // if this is a relative folder path, and the folder is in the cwd, use cwd as the base dir + if (__isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) + absBaseDir = cwd; + else + absBaseDir = find_import_dir(normalPath); - // if this is a relative folder path, and the folder is in the cwd, use cwd as the base dir - if (__isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) { - absBaseDir = cwd; - } else { - absBaseDir = find_import_dir(normalPath); - } + if (absBaseDir.empty()) + return false; - if (absBaseDir.empty()) { - return false; - } + if (!normalPath.is_absolute()) + absPath = FSUtils::canonicalFS(absBaseDir / normalPath); + else + absPath = FSUtils::canonicalFS(normalPath); - if (!normalPath.is_absolute()) { - absPath = FSUtils::canonicalFS(absBaseDir / normalPath); - } else { - absPath = FSUtils::canonicalFS(normalPath); - } - - if (std::filesystem::exists(absPath)) { - resolved = true; - return true; - } + if (!std::filesystem::exists(absPath)) return false; - } + if (__isFolder && !std::filesystem::is_directory(absPath)) + return false; + + resolved = true; + return true; +} } // namespace caprica diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index 665a3e5..4bb26b3 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -1,80 +1,60 @@ #pragma once #include -namespace caprica{ +namespace caprica { struct IInputFile; struct IInputFile { - virtual std::filesystem::path resolved_relative() const = 0; - virtual std::filesystem::path resolved_absolute() const = 0; - virtual std::filesystem::path resolved_absolute_basedir() const = 0; + std::filesystem::path resolved_relative() const; + std::filesystem::path resolved_absolute() const; + std::filesystem::path resolved_absolute_basedir() const; + std::filesystem::path resolved_relative_parent() const; std::filesystem::path get_unresolved_path() const { return rawPath; } bool isRecursive() const { return !noRecurse; } bool isImport() const { return import; } - virtual bool exists() const = 0; - virtual bool isDir() const = 0; + bool isResolved() const { return resolved; } + virtual bool exists() const; + virtual bool isDir() const; virtual bool resolve() = 0; - virtual bool isResolved() const { return resolved; } - IInputFile(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = ""); + IInputFile(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); virtual ~IInputFile() = default; - // static InputFile createNewStyle(const std::filesystem::path& path, bool noRecurse = true, const std::filesystem::path& cwd = std::filesystem::current_path()); - // static InputFile createOldStyle(const std::filesystem::path& path, bool noRecurse = true, const std::filesystem::path& cwd = std::filesystem::current_path(), bool isFolder = false); protected: bool noRecurse = true; const std::filesystem::path rawPath; const std::filesystem::path cwd; + std::filesystem::path absPath; + std::filesystem::path absBaseDir; // import dir + static std::filesystem::path find_import_dir(const std::filesystem::path& _path); + static bool dirContains(const std::filesystem::path& _path, const std::filesystem::path& dir); + bool resolved = false; bool import = false; bool requiresPreParse = false; - static std::filesystem::path find_import_dir(const std::filesystem::path& _path); - static bool dirContains(const std::filesystem::path& _path, const std::filesystem::path& dir); }; struct InputFile : public IInputFile { - virtual std::filesystem::path resolved_relative() const override; - virtual std::filesystem::path resolved_absolute() const override; - virtual std::filesystem::path resolved_absolute_basedir() const override; - virtual bool exists() const override; - virtual bool isDir() const override; virtual bool resolve() override; - InputFile(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = ""); + InputFile(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); virtual ~InputFile() = default; - - private: - std::filesystem::path absPath; - std::filesystem::path absBaseDir; }; struct PCompInputFile : public IInputFile { PCompInputFile(const std::filesystem::path& _path, - bool noRecurse = true, bool isFolder = false, - const std::filesystem::path& _cwd = ""); - virtual std::filesystem::path resolved_relative() const override; - virtual std::filesystem::path resolved_absolute() const override; - virtual std::filesystem::path resolved_absolute_basedir() const override; - virtual bool exists() const override; + bool noRecurse = true, + bool isFolder = false, + const std::filesystem::path& _cwd = ""); virtual bool isDir() const override { return __isFolder; } virtual bool resolve() override; - private: - bool __isFolder = false; - std::filesystem::path absPath; - std::filesystem::path absBaseDir; + +private: + bool __isFolder = false; }; struct ImportDir : public IInputFile { - ImportDir(const std::filesystem::path& _path, - bool noRecurse = true, - const std::filesystem::path& _cwd = ""); - virtual std::filesystem::path resolved_relative() const override; - virtual std::filesystem::path resolved_absolute() const override; - virtual std::filesystem::path resolved_absolute_basedir() const override; - virtual bool exists() const override; + ImportDir(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); virtual bool isDir() const override { return true; } virtual bool resolve() override; - private: - std::filesystem::path resolvedPath; + +private: + std::filesystem::path resolvedPath; }; -} // namespace caprica \ No newline at end of file +} // namespace caprica From e5569c7669f08665d662b5f69411e52172283339 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:02:52 -0700 Subject: [PATCH 20/22] fix stat reporting --- Caprica/common/CapricaStats.cpp | 13 +++++++------ Caprica/common/CapricaStats.h | 24 +++++++++--------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Caprica/common/CapricaStats.cpp b/Caprica/common/CapricaStats.cpp index 85c3eb9..76b497c 100644 --- a/Caprica/common/CapricaStats.cpp +++ b/Caprica/common/CapricaStats.cpp @@ -5,13 +5,13 @@ namespace caprica { -CapricaStats::counter_type CapricaStats::peekedTokenCount { 0 }; -CapricaStats::counter_type CapricaStats::consumedTokenCount { 0 }; -CapricaStats::counter_type CapricaStats::lexedFilesCount { 0 }; +CapricaStats::NopIncStruct CapricaStats::peekedTokenCount { 0 }; +CapricaStats::NopIncStruct CapricaStats::consumedTokenCount { 0 }; +CapricaStats::NopIncStruct CapricaStats::lexedFilesCount { 0 }; CapricaStats::counter_type CapricaStats::importedFileCount { 0 }; CapricaStats::counter_type CapricaStats::inputFileCount { 0 }; -CapricaStats::counter_type CapricaStats::allocatedHeapCount { 0 }; -CapricaStats::counter_type CapricaStats::freedHeapCount { 0 }; +CapricaStats::NopIncStruct CapricaStats::allocatedHeapCount { 0 }; +CapricaStats::NopIncStruct CapricaStats::freedHeapCount { 0 }; template static std::enable_if_t::value> internalOutputStats() { @@ -43,7 +43,8 @@ void CapricaStats::outputStats() { } void CapricaStats::outputImportedCount() { - std::cout << "Imported " << importedFileCount.val << " files." << std::endl; + std::cout << "Imported " << importedFileCount << " files." << std::endl; + std::cout << "Compiling " << inputFileCount << " files..." << std::endl; } } diff --git a/Caprica/common/CapricaStats.h b/Caprica/common/CapricaStats.h index f183195..7c26c08 100644 --- a/Caprica/common/CapricaStats.h +++ b/Caprica/common/CapricaStats.h @@ -9,28 +9,22 @@ struct CapricaStats final { struct NopIncStruct final { size_t val; - NopIncStruct& operator++(int) { - val++; - return *this; - } - NopIncStruct& operator=(size_t f) { - val = f; - return *this; - } + NopIncStruct& operator++(int) { return *this; } + NopIncStruct& operator=(size_t) { return *this; } }; // using counter_type = std::atomic; - // using counter_type = size_t; - using counter_type = NopIncStruct; + using counter_type = size_t; + // using counter_type = NopIncStruct; public: - static counter_type peekedTokenCount; - static counter_type consumedTokenCount; + static NopIncStruct peekedTokenCount; + static NopIncStruct consumedTokenCount; static counter_type importedFileCount; static counter_type inputFileCount; - static counter_type lexedFilesCount; - static counter_type allocatedHeapCount; - static counter_type freedHeapCount; + static NopIncStruct lexedFilesCount; + static NopIncStruct allocatedHeapCount; + static NopIncStruct freedHeapCount; static void outputStats(); static void outputImportedCount(); From c9e91d247fdb76b3948fc127beb5d0ca4368398f Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:22:37 -0700 Subject: [PATCH 21/22] pre-parse all non pcompiler/ppj input files for NS --- Caprica/common/CapricaInputFile.cpp | 33 ++++++-- Caprica/common/CapricaInputFile.h | 1 + Caprica/common/FSUtils.cpp | 1 + Caprica/main.cpp | 76 ++++++++++++------- Caprica/papyrus/PapyrusCompilationContext.cpp | 39 ++++++---- Caprica/papyrus/PapyrusCompilationContext.h | 17 ++++- 6 files changed, 117 insertions(+), 50 deletions(-) diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index 68cbe76..df78232 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -73,12 +73,34 @@ bool IInputFile::isDir() const { return std::filesystem::is_directory(resolved_absolute()); } +bool IInputFile::requiresRemap() const { + return requiresPreParse; +} + +static constexpr char const curDir[3] = { '.', FSUtils::SEP, 0 }; +static constexpr char const parent[4] = { '.', '.', FSUtils::SEP, 0 }; + +std::filesystem::path getCorrectBaseDir(const std::filesystem::path& normalPath, + const std::filesystem::path& absBaseDir) { + // for every 2 ..s in the path, remove a directory from the base dir + auto str = normalPath.string(); + auto ret = absBaseDir; + while (str.starts_with(parent)) { + ret = ret.parent_path(); + str = str.substr(3); + } + return ret; +} + bool InputFile::resolve() { auto normalPath = FSUtils::normalize(rawPath); - auto str = normalPath.string(); if (!normalPath.is_absolute()) { absPath = FSUtils::canonicalFS(cwd / normalPath); - absBaseDir = cwd; + absBaseDir = find_import_dir(absPath); + if (absBaseDir.empty()) { + absBaseDir = getCorrectBaseDir(normalPath, cwd); + requiresPreParse = true; + } if (std::filesystem::exists(absPath)) { resolved = true; return true; @@ -135,9 +157,6 @@ PCompInputFile::PCompInputFile(const std::filesystem::path& _path, __isFolder = isFolder; } -static constexpr char const curDir[3] = { '.', FSUtils::SEP, 0 }; -static constexpr char const parent[4] = { '.', '.', FSUtils::SEP, 0 }; - bool PCompInputFile::resolve() { std::filesystem::path normalPath = FSUtils::objectNameToPath(rawPath.string()); if (!__isFolder && normalPath.extension().empty()) @@ -147,7 +166,7 @@ bool PCompInputFile::resolve() { // special case for relative paths that contain parent/cwd refs if (!normalPath.is_absolute() && (str == "." || str == ".." || str.starts_with(curDir) || str.contains(parent))) { absPath = FSUtils::canonicalFS(cwd / normalPath); - absBaseDir = cwd; + absBaseDir = getCorrectBaseDir(normalPath, cwd); if (!std::filesystem::exists(absPath)) return false; @@ -157,7 +176,7 @@ bool PCompInputFile::resolve() { // if this is a relative folder path, and the folder is in the cwd, use cwd as the base dir if (__isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) - absBaseDir = cwd; + absBaseDir = getCorrectBaseDir(normalPath, cwd); else absBaseDir = find_import_dir(normalPath); diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index 4bb26b3..4ca2375 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -13,6 +13,7 @@ struct IInputFile { bool isResolved() const { return resolved; } virtual bool exists() const; virtual bool isDir() const; + bool requiresRemap() const; virtual bool resolve() = 0; IInputFile(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); virtual ~IInputFile() = default; diff --git a/Caprica/common/FSUtils.cpp b/Caprica/common/FSUtils.cpp index 8af7411..8c2c8c0 100644 --- a/Caprica/common/FSUtils.cpp +++ b/Caprica/common/FSUtils.cpp @@ -62,6 +62,7 @@ bool shouldShortCircuit(const std::string& path) { // Borrowed and modified from http://stackoverflow.com/a/1750710/776797 std::filesystem::path normalize(const std::filesystem::path& path) { #ifdef _WIN32 + if (!path.is_absolute()) return path.lexically_normal(); std::filesystem::path result; for (auto it = path.begin(); it != path.end(); ++it) { if (!wcscmp(it->c_str(), L"..")) { diff --git a/Caprica/main.cpp b/Caprica/main.cpp index 8769dff..2977f0a 100644 --- a/Caprica/main.cpp +++ b/Caprica/main.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -96,7 +97,8 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType const std::filesystem::path& baseOutputDir, const std::filesystem::path& curDir, const std::filesystem::path& absBaseDir, - const WIN32_FIND_DATA& data); + const WIN32_FIND_DATA& data, + bool strictNS); PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType, CapricaJobManager* jobManager, @@ -105,7 +107,8 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType const std::filesystem::path& absBaseDir, const std::filesystem::path& fileName, time_t lastModTime, - size_t fileSize); + size_t fileSize, + bool strictNS); bool addSingleFile(const IInputFile& input, const std::filesystem::path& baseOutputDir, @@ -126,6 +129,7 @@ bool addFilesFromDirectory(const IInputFile& input, std::cout << "ERROR: Input file '" << input.get_unresolved_path() << "' was not found!" << std::endl; return false; } + auto abspath = input.resolved_absolute(); auto absBaseDir = input.resolved_absolute_basedir(); auto subdir = input.resolved_relative(); if (subdir == ".") @@ -135,6 +139,12 @@ bool addFilesFromDirectory(const IInputFile& input, dirs.push_back(subdir); auto baseDirMap = getBaseSigMap(conf::Papyrus::game); auto l_startNS = startingNS; + if (l_startNS == "") { + if (conf::Papyrus::game > GameID::Skyrim) { + if (input.requiresRemap()) + l_startNS = "!!temp" + abspath.filename().string() + std::to_string(rand()); + } + } const auto DOTDOT = std::string_view(".."); const auto DOT = std::string_view("."); @@ -184,7 +194,8 @@ bool addFilesFromDirectory(const IInputFile& input, break; } if (!skip) { - PapyrusCompilationNode* node = getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data); + PapyrusCompilationNode* node = + getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data, !input.requiresRemap()); namespaceMap.emplace(caprica::identifier_ref(node->baseName), node); if (!gBaseFound && !baseDirMap.empty()) { @@ -236,7 +247,8 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType const std::filesystem::path& absBaseDir, const std::filesystem::path& fileName, time_t lastModTime, - size_t fileSize) { + size_t fileSize, + bool strictNS) { std::filesystem::path cur; if (curDir == ".") cur = ""; @@ -244,7 +256,6 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType cur = curDir; std::filesystem::path sourceFilePath = absBaseDir / cur / fileName; std::filesystem::path filenameToDisplay = cur / fileName; - std::filesystem::path outputDir = baseOutputDir / cur; if (nodeType == PapyrusCompilationNode::NodeType::PapyrusImport || nodeType == PapyrusCompilationNode::NodeType::PasReflection || @@ -256,10 +267,11 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType auto node = new caprica::papyrus::PapyrusCompilationNode(jobManager, nodeType, std::move(filenameToDisplay.string()), - std::move(outputDir.string()), + std::move(baseOutputDir.string()), std::move(sourceFilePath.string()), lastModTime, - fileSize); + fileSize, + strictNS); return node; } @@ -268,7 +280,8 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType const std::filesystem::path& baseOutputDir, const std::filesystem::path& curDir, const std::filesystem::path& absBaseDir, - const WIN32_FIND_DATA& data) { + const WIN32_FIND_DATA& data, + bool strictNS) { const auto lastModTime = [](FILETIME ft) -> time_t { ULARGE_INTEGER ull; ull.LowPart = ft.dwLowDateTime; @@ -281,7 +294,15 @@ PapyrusCompilationNode* getNode(const PapyrusCompilationNode::NodeType& nodeType ull.HighPart = high; return ull.QuadPart; }(data.nFileSizeLow, data.nFileSizeHigh); - return getNode(nodeType, jobManager, baseOutputDir, curDir, absBaseDir, data.cFileName, lastModTime, fileSize); + return getNode(nodeType, + jobManager, + baseOutputDir, + curDir, + absBaseDir, + data.cFileName, + lastModTime, + fileSize, + strictNS); } bool handleImports(const std::vector& f, caprica::CapricaJobManager* jobManager) { @@ -290,15 +311,15 @@ bool handleImports(const std::vector& f, caprica::CapricaJobManager* caprica::caseless_unordered_identifier_ref_map tempMap{}; for (auto &fake_script: FAKE_SKYRIM_SCRIPTS_SET) { auto basename = caprica::FSUtils::filenameAsRef(fake_script); - auto node = - new PapyrusCompilationNode(jobManager, - PapyrusCompilationNode::NodeType::PapyrusImport, - std::string(basename), - "", - std::string(fake_script), - 0, - caprica::FakeScripts::getSizeOfFakeScript(fake_script, conf::Papyrus::game)); + new PapyrusCompilationNode(jobManager, + PapyrusCompilationNode::NodeType::PapyrusImport, + std::string(basename), + "", + std::string(fake_script), + 0, + caprica::FakeScripts::getSizeOfFakeScript(fake_script, conf::Papyrus::game), + false); tempMap.emplace(caprica::identifier_ref(node->baseName), node); } caprica::papyrus::PapyrusCompilationContext::pushNamespaceFullContents("", std::move(tempMap)); @@ -312,8 +333,6 @@ bool handleImports(const std::vector& f, caprica::CapricaJobManager* if (!addFilesFromDirectory(dir, "", jobManager, PapyrusCompilationNode::NodeType::PapyrusImport, ns)) return false; } - CapricaStats::outputImportedCount(); - caprica::papyrus::PapyrusCompilationContext::RenameImports(jobManager); return true; } @@ -329,11 +348,6 @@ bool addSingleFile(const IInputFile& input, const auto absPath = input.resolved_absolute(); const auto filename = absPath.filename(); const auto absBaseDir = input.resolved_absolute_basedir(); - if (!input.exists()) { - std::cout << "ERROR: Resolved file path '" << absPath - << "' is not in an import directory, cannot resolve namespace!" << std::endl; - return false; - } auto lastModTime = std::filesystem::last_write_time(absPath, ec); if (ec) { std::cout << "An error occurred while trying to get the last modified time of '" << absPath << "'!" << std::endl; @@ -346,8 +360,11 @@ bool addSingleFile(const IInputFile& input, } auto namespaceName = FSUtils::pathToObjectName(namespaceDir); - if (!conf::General::quietCompile) - std::cout << "Adding file '" << filename << "' to namespace '" << namespaceName << "'." << std::endl; + if (conf::Papyrus::game > GameID::Skyrim) { + if (input.requiresRemap()) + namespaceName = "!!temp" + absPath.filename().string() + std::to_string(rand()) + ":" + namespaceName; + } + auto node = getNode(nodeType, jobManager, baseOutputDir, @@ -355,7 +372,8 @@ bool addSingleFile(const IInputFile& input, absBaseDir, filename, lastModTime.time_since_epoch().count(), - fileSize); + fileSize, + !input.requiresRemap()); caprica::papyrus::PapyrusCompilationContext::pushNamespaceFullContents( namespaceName, caprica::caseless_unordered_identifier_ref_map{ @@ -382,6 +400,10 @@ int main(int argc, char *argv[]) { } if (conf::General::compileInParallel) jobManager.startup((uint32_t) std::thread::hardware_concurrency()); + + caprica::papyrus::PapyrusCompilationContext::RenameImports(&jobManager); + caprica::CapricaStats::outputImportedCount(); + auto endParse = std::chrono::high_resolution_clock::now(); if (conf::Performance::dumpTiming) { std::cout << "Command Line Arg Parse: " diff --git a/Caprica/papyrus/PapyrusCompilationContext.cpp b/Caprica/papyrus/PapyrusCompilationContext.cpp index e2c43db..493b97e 100644 --- a/Caprica/papyrus/PapyrusCompilationContext.cpp +++ b/Caprica/papyrus/PapyrusCompilationContext.cpp @@ -26,7 +26,6 @@ std::string PapyrusCompilationNode::awaitPreParse() { return objectName; } - PapyrusScript* PapyrusCompilationNode::awaitParse() { parseJob.await(); return loadedScript; @@ -158,30 +157,44 @@ void PapyrusCompilationNode::FilePreParseJob::run() { } if (parent->objectName.empty()) CapricaReportingContext::logicalFatal("Unable to find script name in '{}'.", parent->sourceFilePath); +} +void PapyrusCompilationNode::FileParseJob::run() { + parent->preParseJob.await(); // Check if we have the correct namespace + switch (parent->type) { case NodeType::PapyrusCompile: // only check for this on compile nodes + case NodeType::PexDissassembly: case NodeType::PasCompile: { // check the object name with the reportedname auto nsName = FSUtils::pathToObjectName(parent->reportedName); if (_strnicmp(parent->objectName.c_str(), nsName.c_str(), parent->objectName.size()) != 0) { - CapricaReportingContext::logicalFatal( - "{}: The script namespace '{}' does not match the expected namespace '{}'.\n" - "Check your imports and your CWD.", - parent->reportedName, - parent->objectName, - nsName); + if (parent->strictNS) { + CapricaReportingContext::logicalFatal( + "{}: The script namespace '{}' does not match the expected namespace '{}'.\n" + "Check your imports and your CWD.", + parent->reportedName, + parent->objectName, + nsName); + } else { + // change the outputDir + auto newPath = FSUtils::objectNameToPath(parent->objectName); + auto pardir = newPath.parent_path(); + parent->outputDirectory = (parent->baseOutputDirectory / pardir).string(); + + // // change the reported name + // auto ext = FSUtils::extensionAsRef(parent->sourceFilePath); + // parent->reportedName = newPath.replace_extension(ext).string(); + // TODO: emit warning? + } } } break; default: // We don't check for this on other jobs; imports will likely have different names than their reported // names break; } -} -void PapyrusCompilationNode::FileParseJob::run() { - parent->preParseJob.await(); auto ext = FSUtils::extensionAsRef(parent->sourceFilePath); if (pathEq(ext, ".psc")) { auto parser = new parser::PapyrusParser(parent->reportingContext, parent->sourceFilePath, parent->readFileData); @@ -321,7 +334,8 @@ void PapyrusCompilationNode::FileWriteJob::run() { auto containingDir = std::filesystem::path(parent->outputDirectory); if (!std::filesystem::exists(containingDir)) std::filesystem::create_directories(containingDir); - std::ofstream destFile { parent->outputDirectory + FSUtils::SEP + baseFileName + ".pex", std::ifstream::binary }; + std::ofstream destFile { parent->outputDirectory + FSUtils::SEP + baseFileName + ".pex", + std::ifstream::binary }; destFile.exceptions(std::ifstream::badbit | std::ifstream::failbit); parent->pexWriter->applyToBuffers([&](const char* data, size_t size) { destFile.write(data, size); }); } @@ -588,9 +602,8 @@ void PapyrusCompilationContext::RenameImports(CapricaJobManager* jobManager) { continue; rootNamespace.children.erase(it); it = rootNamespace.children.begin(); - if (it == rootNamespace.children.end()){ + if (it == rootNamespace.children.end()) break; - } } } diff --git a/Caprica/papyrus/PapyrusCompilationContext.h b/Caprica/papyrus/PapyrusCompilationContext.h index 49530e0..592f40b 100644 --- a/Caprica/papyrus/PapyrusCompilationContext.h +++ b/Caprica/papyrus/PapyrusCompilationContext.h @@ -34,15 +34,24 @@ struct PapyrusCompilationNode final { std::string&& baseOutputDir, std::string&& absolutePath, time_t lastMod, - size_t fileSize) + size_t fileSize, + bool strictNS) : reportedName(std::move(sourcePath)), - outputDirectory(std::move(baseOutputDir)), + baseOutputDirectory(std::move(baseOutputDir)), sourceFilePath(std::move(absolutePath)), lastModTime(lastMod), filesize(fileSize), reportingContext(reportedName), jobManager(mgr), - type(compileType) { + type(compileType), + strictNS(strictNS) { + + // set the output directory + if (reportedName.find_last_of("\\/") != std::string::npos) + outputDirectory = baseOutputDirectory + FSUtils::SEP + std::string(FSUtils::parentPathAsRef(reportedName)); + else + outputDirectory = baseOutputDirectory; + baseName = FSUtils::basenameAsRef(sourceFilePath); // TODO: fix Imports hack if (type == NodeType::PapyrusImport) @@ -90,6 +99,8 @@ struct PapyrusCompilationNode final { std::string reportedName; std::string objectName; bool isPexFile = false; + bool strictNS = false; + std::string baseOutputDirectory; std::string outputDirectory; std::string sourceFilePath; std::string_view readFileData {}; From c4950d1093712cf10c4844737705fed54007a0cc Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 13 Oct 2023 23:17:56 -0700 Subject: [PATCH 22/22] don't recheck fs for inputfile::isDir --- Caprica/common/CapricaInputFile.cpp | 38 +++++++++-------------------- Caprica/common/CapricaInputFile.h | 12 +++------ 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/Caprica/common/CapricaInputFile.cpp b/Caprica/common/CapricaInputFile.cpp index df78232..abaa368 100644 --- a/Caprica/common/CapricaInputFile.cpp +++ b/Caprica/common/CapricaInputFile.cpp @@ -69,17 +69,8 @@ bool IInputFile::exists() const { return std::filesystem::exists(resolved_absolute()); } -bool IInputFile::isDir() const { - return std::filesystem::is_directory(resolved_absolute()); -} - -bool IInputFile::requiresRemap() const { - return requiresPreParse; -} - static constexpr char const curDir[3] = { '.', FSUtils::SEP, 0 }; static constexpr char const parent[4] = { '.', '.', FSUtils::SEP, 0 }; - std::filesystem::path getCorrectBaseDir(const std::filesystem::path& normalPath, const std::filesystem::path& absBaseDir) { // for every 2 ..s in the path, remove a directory from the base dir @@ -92,6 +83,11 @@ std::filesystem::path getCorrectBaseDir(const std::filesystem::path& normalPath, return ret; } +InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) + : IInputFile(_path, noRecurse, _cwd) { + requiresPreParse = true; // we always require pre-parse for non-PCompiler-compatible input files +} + bool InputFile::resolve() { auto normalPath = FSUtils::normalize(rawPath); if (!normalPath.is_absolute()) { @@ -99,24 +95,18 @@ bool InputFile::resolve() { absBaseDir = find_import_dir(absPath); if (absBaseDir.empty()) { absBaseDir = getCorrectBaseDir(normalPath, cwd); - requiresPreParse = true; - } - if (std::filesystem::exists(absPath)) { - resolved = true; - return true; - } else { - return false; } } else { absPath = FSUtils::canonicalFS(normalPath); absBaseDir = find_import_dir(absPath); if (absBaseDir.empty()) { absBaseDir = absPath.parent_path(); - requiresPreParse = true; } } if (std::filesystem::exists(absPath)) { + if (std::filesystem::is_directory(absPath)) + isFolder = true; resolved = true; return true; } @@ -124,15 +114,11 @@ bool InputFile::resolve() { return false; } -InputFile::InputFile(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) - : IInputFile(_path, noRecurse, _cwd) { - requiresPreParse = true; // we always require pre-parse for non-PCompiler-compatible input files -} - ImportDir::ImportDir(const std::filesystem::path& _path, bool noRecurse, const std::filesystem::path& _cwd) : IInputFile(_path, noRecurse, _cwd) { requiresPreParse = true; // we always require pre-parse for import dirs import = true; + isFolder = true; resolve(); // we resolve import dirs immediately } @@ -154,12 +140,12 @@ PCompInputFile::PCompInputFile(const std::filesystem::path& _path, bool isFolder, const std::filesystem::path& _cwd) : IInputFile(_path, noRecurse, _cwd) { - __isFolder = isFolder; + isFolder = isFolder; } bool PCompInputFile::resolve() { std::filesystem::path normalPath = FSUtils::objectNameToPath(rawPath.string()); - if (!__isFolder && normalPath.extension().empty()) + if (!isFolder && normalPath.extension().empty()) normalPath.replace_extension(".psc"); normalPath = FSUtils::normalize(normalPath); std::string str = normalPath.string(); @@ -175,7 +161,7 @@ bool PCompInputFile::resolve() { } // if this is a relative folder path, and the folder is in the cwd, use cwd as the base dir - if (__isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) + if (isFolder && !normalPath.is_absolute() && dirContains(normalPath, cwd)) absBaseDir = getCorrectBaseDir(normalPath, cwd); else absBaseDir = find_import_dir(normalPath); @@ -190,7 +176,7 @@ bool PCompInputFile::resolve() { if (!std::filesystem::exists(absPath)) return false; - if (__isFolder && !std::filesystem::is_directory(absPath)) + if (isFolder && !std::filesystem::is_directory(absPath)) return false; resolved = true; diff --git a/Caprica/common/CapricaInputFile.h b/Caprica/common/CapricaInputFile.h index 4ca2375..4fc6ff7 100644 --- a/Caprica/common/CapricaInputFile.h +++ b/Caprica/common/CapricaInputFile.h @@ -11,9 +11,9 @@ struct IInputFile { bool isRecursive() const { return !noRecurse; } bool isImport() const { return import; } bool isResolved() const { return resolved; } - virtual bool exists() const; - virtual bool isDir() const; - bool requiresRemap() const; + bool isDir() const { return isFolder; } + bool exists() const; + bool requiresRemap() const { return requiresPreParse; } virtual bool resolve() = 0; IInputFile(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); virtual ~IInputFile() = default; @@ -30,6 +30,7 @@ struct IInputFile { bool resolved = false; bool import = false; bool requiresPreParse = false; + bool isFolder = false; }; struct InputFile : public IInputFile { @@ -43,16 +44,11 @@ struct PCompInputFile : public IInputFile { bool noRecurse = true, bool isFolder = false, const std::filesystem::path& _cwd = ""); - virtual bool isDir() const override { return __isFolder; } virtual bool resolve() override; - -private: - bool __isFolder = false; }; struct ImportDir : public IInputFile { ImportDir(const std::filesystem::path& _path, bool noRecurse = true, const std::filesystem::path& _cwd = ""); - virtual bool isDir() const override { return true; } virtual bool resolve() override; private: