From 36a948573fdd0b2c29f3c4418ebd5e59f454227a Mon Sep 17 00:00:00 2001 From: jnack <55568980+jnackmclain@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:03:09 -0500 Subject: [PATCH 1/2] anyway just make a python directory in mhx/tools for now --- python/build_template/configure_build.py | 349 +++++++++++++++++++++++ python/build_template/gen_version.py | 18 ++ python/build_template/png_list.py | 15 + 3 files changed, 382 insertions(+) create mode 100644 python/build_template/configure_build.py create mode 100644 python/build_template/gen_version.py create mode 100644 python/build_template/png_list.py diff --git a/python/build_template/configure_build.py b/python/build_template/configure_build.py new file mode 100644 index 0000000..575411f --- /dev/null +++ b/python/build_template/configure_build.py @@ -0,0 +1,349 @@ +#!/usr/bin/python3 +from lib import ninja_syntax +from pathlib import Path +import sys +import argparse +import os +import dx_build + +game_name = "Rock Band 3 Deluxe" + +ninja = ninja_syntax.Writer(open("build.ninja", "w+")) + +#versions +#gh1, gh2 - " " +#rb1, track packs - "-v 4" +#rb2, tbrb, gdrb - "-v 5" +#rb3, dc, blitz - "-v 6" +ninja.variable("ark_version", "-v 6") +ark_version = "-v 6" + +#require user provided vanilla ark extract in "vanilla/platform" folder +vanilla_files = False + +#set True for rb1 and newer +new_gen = True + +#patch for patch arks, main for patchcreator, old gen, and rb1 +hdr_name = "patch" + +dtb_encrypt = "-e" +ark_encrypt = "-e" +miloVersion = "--miloVersion 26" + +#paths in _ark/dx/custom_textures that should generate list dtbs +custom_texture_paths = [ + "highways", + "streaks", + "overdrive", + "gems/gems_default", + "strikeline/strikeline_guitar", + "flames/flames_spark", + "sustains", + "score/scoreboard_frame", + "rails/beat_lines", + "stars/score_star_frame", + "font", + "solo_box", + "bre/bre_shield", + "rails/rails_track", + "lanes/gem_mash_green_emmisive", + "overdrive_bar/od_bar_background", + "multiplier_ring/multiplier_ring_plate_fc", + "crowd_meter/crowd_meter_frame", + "keyboard/keyboard_lanes", + "vocal_highway/vocal_highway_bg", + "vocal_arrows/vocal_arrow", + "vocal_note/vocal_note_tube", + "vocal_overdrive/vocal_overdrive_now_bar" +] + +#patchcreator options +patchcreator = False +new_ark_part = "10" +dol_name = "main.dol" +elf_name = " " +bin_name = " " +xex_name = " " + +parser = argparse.ArgumentParser(prog="configure") +parser.add_argument("platform") +parser.add_argument( + "--no-updates", action="store_true", help="disable dx song updates" +) + +args = parser.parse_args() + +#THIS IS TEMPLATE YOU CAN REMOVE IF YOU DONT NEED IT +if args.platform == "ps2": + #all milo ps2 games from gh to rb use MAIN_0.ARK + hdr_name = "MAIN" + if new_gen == False: + #pre rb2 does not use miloVersion for superfreq image generation on ps2 + miloVersion = " " + #remove these two if rock band 1 + ark_encrypt = " " + dtb_encrypt = dtb_encrypt.upper() + +ninja.variable("dtb_encrypt", dtb_encrypt) +ninja.variable("ark_encrypt", ark_encrypt) +ninja.variable("miloVersion", miloVersion) + +#new gen games (rb2 onward) add platform suffix to ark name +if new_gen == True: + hdr_name = hdr_name + "_" + args.platform + +print(f"Configuring {game_name}...") +print(f"Platform: {args.platform}") + +# configure tools +ark_dir = Path("obj", args.platform, "ark") +match sys.platform: + case "win32": + ninja.variable("silence", ">nul") + ninja.rule("copy", "cmd /c copy $in $out $silence", description="COPY $in") + ninja.rule("bswap", "dependencies\\windows\\swap_art_bytes.exe $in $out", description="BSWAP $in") + ninja.rule("version", "python dependencies\\python\\gen_version.py $out", description="Writing version info") + ninja.rule("png_list", "python dependencies\\python\\png_list.py $dir $out", description="PNGLIST $dir") + ninja.variable("superfreq", "dependencies\\windows\\superfreq.exe") + ninja.variable("arkhelper", "dependencies\\windows\\arkhelper.exe") + ninja.variable("dtab", "dependencies\\windows\\dtab.exe") + ninja.variable("dtacheck", "dependencies\\windows\\dtacheck.exe") + case "darwin": + ninja.variable("silence", "> /dev/null") + ninja.rule("copy", "cp $in $out", description="COPY $in") + ninja.rule("bswap", "python3 dependencies/python/swap_rb_art_bytes.py $in $out", description="BSWAP $in") + ninja.rule("version", "python3 dependencies/python/gen_version.py $out", description="Writing version info") + ninja.rule("png_list", "python3 dependencies/python/png_list.py $dir $out", description="PNGLIST $dir") + ninja.variable("superfreq", "dependencies/macos/superfreq") + ninja.variable("arkhelper", "dependencies/macos/arkhelper") + ninja.variable("dtab", "dependencies/macos/dtab") + # dtacheck needs to be compiled for mac + ninja.variable("dtacheck", "true") + case "linux": + ninja.variable("silence", "> /dev/null") + ninja.rule("copy", "cp --reflink=auto $in $out",description="COPY $in") + ninja.rule("bswap", "dependencies/linux/swap_art_bytes $in $out", "BSWAP $in") + ninja.rule("version", "python dependencies/python/gen_version.py $out", description="Writing version info") + ninja.rule("png_list", "python dependencies/python/png_list.py $dir $out", description="PNGLIST $dir") + ninja.variable("superfreq", "dependencies/linux/superfreq") + ninja.variable("arkhelper", "dependencies/linux/arkhelper") + ninja.variable("dtab", "dependencies/linux/dtab") + ninja.variable("dtacheck", "dependencies/linux/dtacheck") + +#specify output directories per platform +match args.platform: + case "ps3": + out_dir = Path("out", args.platform, "USRDIR", "gen") + case "xbox": + out_dir = Path("out", args.platform, "gen") + case "wii": + out_dir = Path("out", args.platform, "files") + case "ps2": + out_dir = Path("out", args.platform, "GEN") + +#building an ark +if args.platform in ["ps3", "xbox", "ps2"] and patchcreator == False: + ninja.rule( + "ark", + f"$arkhelper dir2ark -n {hdr_name} $ark_version $ark_encrypt -s 4073741823 --logLevel error {ark_dir} {out_dir}", + description="Building ark", + ) +#patchcreating an ark +if args.platform == "wii" or patchcreator == True: + #patch creator time! force using main as the root name + hdr_name = "main" + #append platform if this is new style ark + if new_gen == True: + hdr_name = hdr_name + "_" + args.platform + match args.platform: + case "wii": + hdr_path = "platform/" + args.platform + "/files/gen/" + hdr_name + ".hdr" + exec_path = "platform/" + args.platform + "/sys/" + dol_name + case "ps2": + hdr_path = "platform/" + args.platform + "/GEN/" + hdr_name.upper() + ".HDR" + exec_path = "platform/" + args.platform + "/" + elf_name + case "ps3": + hdr_path = "platform/" + args.platform + "/USRDIR/gen/" + hdr_name + ".hdr" + exec_path = "platform/" + args.platform + "/USRDIR/" + bin_name + case "xbox": + hdr_path = "platform/" + args.platform + "/gen/" + hdr_name + ".hdr" + exec_path = "platform/" + args.platform + "/" + xex_name + ninja.rule( + "ark", + f"$arkhelper patchcreator -a {ark_dir} -o {out_dir} {hdr_path} {exec_path} --logLevel error", + description="Building ark", + ) + +ninja.rule( + "sfreq", + f"$superfreq png2tex -l error $miloVersion --platform $platform $in $out", + description="SFREQ $in" + ) +ninja.rule("dtacheck", "$dtacheck $in .dtacheckfns", description="DTACHECK $in") +ninja.rule("dtab_serialize", "$dtab -b $in $out", description="DTAB SER $in") +ninja.rule("dtab_encrypt", f"$dtab $dtb_encrypt $in $out", description="DTAB ENC $in") +ninja.build("_always", "phony") + +build_files = [] + +# copy platform files +if args.platform != "wii" and patchcreator == False: + for f in filter(lambda x: x.is_file(), Path("platform", args.platform).rglob("*")): + index = f.parts.index(args.platform) + out_path = Path("out", args.platform).joinpath(*f.parts[index + 1 :]) + ninja.build(str(out_path), "copy", str(f)) + build_files.append(str(out_path)) + +def ark_file_filter(file: Path): + if file.is_dir(): + return False + if file.suffix.endswith("_ps3") and args.platform != "ps3": + return False + if file.suffix.endswith("_xbox") and args.platform != "xbox": + return False + if file.suffix.endswith("_wii") and args.platform != "wii": + return False + if file.suffix.endswith("mogg") and args.platform == "ps2": + return False + if any(file.suffix.endswith(suffix) for suffix in ["_ps2", "vgs"]) and args.platform != "ps2": + return False + if (args.platform == "wii" or args.no_updates) and file.parts[slice(2)] == ("_ark", "songs"): + return False + + return True + +# build ark files +ark_files = [] + +for f in filter(ark_file_filter, Path("_ark").rglob("*")): + match f.suffixes: + case [".png"]: + output_directory = Path("obj", args.platform, "ark").joinpath( + *f.parent.parts[1:] + ) + match args.platform: + case "ps3": + target_filename = Path("gen", f.stem + ".png_ps3") + xbox_filename = Path("gen", f.stem + ".png_xbox") + xbox_directory = Path("obj", args.platform, "raw").joinpath( + *f.parent.parts[1:] + ) + xbox_output = xbox_directory.joinpath(xbox_filename) + ps3_output = output_directory.joinpath(target_filename) + ninja.build(str(xbox_output), "sfreq", str(f), variables={"platform": "x360"}) + ninja.build(str(ps3_output), "bswap", str(xbox_output)) + ark_files.append(str(ps3_output)) + case "xbox": + target_filename = Path("gen", f.stem + ".png_xbox") + xbox_directory = Path("obj", args.platform, "ark").joinpath( + *f.parent.parts[1:] + ) + xbox_output = xbox_directory.joinpath(target_filename) + ninja.build(str(xbox_output), "sfreq", str(f), variables={"platform": "x360"}) + ark_files.append(str(xbox_output)) + case "wii": + target_filename = Path("gen", f.stem + ".png_wii") + wii_directory = Path("obj", args.platform, "ark").joinpath( + *f.parent.parts[1:] + ) + wii_output = wii_directory.joinpath(target_filename) + ninja.build(str(wii_output), "sfreq", str(f), variables={"platform": "wii"}) + ark_files.append(str(wii_output)) + #disabled as we need resizing + #case "ps2": + # target_filename = Path("gen", f.stem + ".png_ps2") + # ps2_directory = Path("obj", args.platform, "ark").joinpath( + # *f.parent.parts[1:] + # ) + # ps2_output = ps2_directory.joinpath(target_filename) + # ninja.build(str(ps2_output), "sfreq", str(f), variables={"platform": "ps2"}) + # ark_files.append(str(ps2_output)) + + case [".dta"]: + target_filename = Path("gen", f.stem + ".dtb") + stamp_filename = Path("gen", f.stem + ".dtb.checked") + + output_directory = Path("obj", args.platform, "ark").joinpath( + *f.parent.parts[1:] + ) + serialize_directory = Path("obj", args.platform, "raw").joinpath( + *f.parent.parts[1:] + ) + + serialize_output = serialize_directory.joinpath(target_filename) + encryption_output = output_directory.joinpath(target_filename) + stamp = serialize_directory.joinpath(stamp_filename) + ninja.build(str(stamp), "dtacheck", str(f)) + ninja.build( + str(serialize_output), + "dtab_serialize", + str(f), + implicit=[str(stamp), "_always"], + ) + ninja.build(str(encryption_output), "dtab_encrypt", str(serialize_output)) + ark_files.append(str(encryption_output)) + case _: + index = f.parts.index("_ark") + out_path = Path("obj", args.platform, "ark").joinpath(*f.parts[index + 1 :]) + ninja.build(str(out_path), "copy", str(f)) + ark_files.append(str(out_path)) + +# write version info +dta = Path("obj", args.platform, "raw", "dx", "locale", "dx_version.dta") +dtb = Path("obj", args.platform, "raw", "dx", "locale", "gen", "dx_version.dtb") +enc = Path("obj", args.platform, "ark", "dx", "locale", "gen", "dx_version.dtb") + +ninja.build(str(dta), "version", implicit="_always") +ninja.build(str(dtb), "dtab_serialize", str(dta)) +ninja.build(str(enc), "dtab_encrypt", str(dtb)) + +ark_files.append(str(enc)) + +#THIS IS TEMPLATE YOU CAN REMOVE IF YOU DONT NEED IT +#copy vanilla files to obj if required +if vanilla_files == True and patchcreator == False: + from vanilla_files import vanilla + ark_files, vanilla_files = vanilla(ark_files, args.platform, ninja, Path) + +def generate_texture_list(input_path: Path): + base = input_path.parts[1:] + dta = Path("obj", args.platform, "raw").joinpath(*base).joinpath("_list.dta") + dtb = Path("obj", args.platform, "raw").joinpath(*base).joinpath("gen", "_list.dtb") + enc = Path("obj", args.platform, "ark").joinpath(*base).joinpath("gen", "_list.dtb") + ninja.build(str(dta), "png_list", variables={"dir": str(input_path)}, implicit="_always") + ninja.build(str(dtb), "dtab_serialize", str(dta)) + ninja.build(str(enc), "dtab_encrypt", str(dtb)) + +root_path = Path("_ark", "dx", "custom_textures") +for texture_list_path in [root_path.joinpath(path) for path in custom_texture_paths]: + generate_texture_list(texture_list_path) + +# build ark +ark_part = "0" +if patchcreator == True: + ark_part = new_ark_part +match args.platform: + case "ps3": + hdr = str(Path("out", args.platform, "USRDIR", hdr_name + ".hdr")) + ark = str(Path("out", args.platform, "USRDIR", hdr_name + "_" + ark_part + ".ark")) + case "xbox": + hdr = str(Path("out", args.platform, hdr_name + ".hdr")) + ark = str(Path("out", args.platform, hdr_name + "_" + ark_part + ".ark")) + case "wii": + hdr = str(Path("out", args.platform, "files", hdr_name + ".hdr")) + ark = str(Path("out", args.platform, "files", hdr_name + "_" + ark_part + ".ark")) + case "ps2": + hdr = str(Path("out", args.platform, hdr_name + ".HDR")) + ark = str(Path("out", args.platform, hdr_name + "_" + ark_part + ".ARK")) +ninja.build( + ark, + "ark", + implicit=ark_files, + implicit_outputs=[hdr], +) +build_files.append(hdr) + +# make the all target build everything +ninja.build("all", "phony", build_files) +ninja.close() \ No newline at end of file diff --git a/python/build_template/gen_version.py b/python/build_template/gen_version.py new file mode 100644 index 0000000..17413f2 --- /dev/null +++ b/python/build_template/gen_version.py @@ -0,0 +1,18 @@ +# add_devbuild.py +from pathlib import Path +import subprocess +import sys + +commit = subprocess.check_output(["git", "describe", "--always", "--dirty"],text=True).strip("\n") +version = f"1.1.0-nightly+{commit}" + +path = sys.argv[1] + +f = open(path, "w") + +f.write(f'(message_motd "Rock Band 3 Deluxe {version} Loaded! Thanks for playing!")\n') +f.write(f'(message_motd_signin "Rock Band 3 Deluxe {version} Loaded! Thanks for playing!")\n') +f.write(f'(message_motd_noconnection "Rock Band 3 Deluxe {version} Loaded! Thanks for playing!")\n') +f.write(f'(rb3e_mod_string "RB3DX {version}")\n') + +f.close() \ No newline at end of file diff --git a/python/build_template/png_list.py b/python/build_template/png_list.py new file mode 100644 index 0000000..10d0acc --- /dev/null +++ b/python/build_template/png_list.py @@ -0,0 +1,15 @@ +from pathlib import Path +import sys + +directory = Path(sys.argv[1]) + +if not directory.is_dir(): + raise Exception("not a directory") + +out_path = sys.argv[2] + +f = open(out_path, "w") +for file in sorted(directory.glob("*")): + f.write(f"'{file.stem}'\n") + +f.close() \ No newline at end of file From d5b92cd96f6f96bff2ae44ea7825745ce5604a9a Mon Sep 17 00:00:00 2001 From: jnack <55568980+jnackmclain@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:17:23 -0500 Subject: [PATCH 2/2] updates --- python/build_template/configure_build.py | 3 +- python/build_template/lib/ninja_syntax.py | 199 ++++++++++++++++++++++ 2 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 python/build_template/lib/ninja_syntax.py diff --git a/python/build_template/configure_build.py b/python/build_template/configure_build.py index 575411f..42966f1 100644 --- a/python/build_template/configure_build.py +++ b/python/build_template/configure_build.py @@ -16,7 +16,6 @@ #rb2, tbrb, gdrb - "-v 5" #rb3, dc, blitz - "-v 6" ninja.variable("ark_version", "-v 6") -ark_version = "-v 6" #require user provided vanilla ark extract in "vanilla/platform" folder vanilla_files = False @@ -83,7 +82,7 @@ miloVersion = " " #remove these two if rock band 1 ark_encrypt = " " - dtb_encrypt = dtb_encrypt.upper() + dtb_encrypt = "-E" ninja.variable("dtb_encrypt", dtb_encrypt) ninja.variable("ark_encrypt", ark_encrypt) diff --git a/python/build_template/lib/ninja_syntax.py b/python/build_template/lib/ninja_syntax.py new file mode 100644 index 0000000..ca73b5b --- /dev/null +++ b/python/build_template/lib/ninja_syntax.py @@ -0,0 +1,199 @@ +#!/usr/bin/python + +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. +""" + +import re +import textwrap + +def escape_path(word): + return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') + +class Writer(object): + def __init__(self, output, width=78): + self.output = output + self.width = width + + def newline(self): + self.output.write('\n') + + def comment(self, text): + for line in textwrap.wrap(text, self.width - 2, break_long_words=False, + break_on_hyphens=False): + self.output.write('# ' + line + '\n') + + def variable(self, key, value, indent=0): + if value is None: + return + if isinstance(value, list): + value = ' '.join(filter(None, value)) # Filter out empty strings. + self._line('%s = %s' % (key, value), indent) + + def pool(self, name, depth): + self._line('pool %s' % name) + self.variable('depth', depth, indent=1) + + def rule(self, name, command, description=None, depfile=None, + generator=False, pool=None, restat=False, rspfile=None, + rspfile_content=None, deps=None): + self._line('rule %s' % name) + self.variable('command', command, indent=1) + if description: + self.variable('description', description, indent=1) + if depfile: + self.variable('depfile', depfile, indent=1) + if generator: + self.variable('generator', '1', indent=1) + if pool: + self.variable('pool', pool, indent=1) + if restat: + self.variable('restat', '1', indent=1) + if rspfile: + self.variable('rspfile', rspfile, indent=1) + if rspfile_content: + self.variable('rspfile_content', rspfile_content, indent=1) + if deps: + self.variable('deps', deps, indent=1) + + def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, + variables=None, implicit_outputs=None, pool=None, dyndep=None): + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append('|') + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append('||') + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) + for x in as_list(implicit_outputs)] + out_outputs.append('|') + out_outputs.extend(implicit_outputs) + + self._line('build %s: %s' % (' '.join(out_outputs), + ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path): + self._line('include %s' % path) + + def subninja(self, path): + self._line('subninja %s' % path) + + def default(self, paths): + self._line('default %s' % ' '.join(as_list(paths))) + + def _count_dollars_before_index(self, s, i): + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == '$': + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text, indent=0): + """Write 'text' word-wrapped at self.width characters.""" + leading_space = ' ' * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(' $') + space = available_space + while True: + space = text.rfind(' ', 0, space) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + + if space < 0: + # No such space; just use the first unescaped space we can find. + space = available_space - 1 + while True: + space = text.find(' ', space + 1) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + ' $\n') + text = text[space+1:] + + # Subsequent lines are continuations, so indent them. + leading_space = ' ' * (indent+2) + + self.output.write(leading_space + text + '\n') + + def close(self): + self.output.close() + + +def as_list(input): + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string): + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert '\n' not in string, 'Ninja syntax does not allow newlines' + # We only have one special metacharacter: '$'. + return string.replace('$', '$$') + + +def expand(string, vars, local_vars={}): + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m): + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string)