From a1fa8b67572d53679adc177f89448d21ad06d38c Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Sat, 16 Nov 2024 16:24:59 +1000 Subject: [PATCH 1/6] Add fallback parsing to ParseStarInput --- PyRoute/Inputs/ParseStarInput.py | 36 ++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/PyRoute/Inputs/ParseStarInput.py b/PyRoute/Inputs/ParseStarInput.py index e6cc2d701..1ad1d4a0c 100644 --- a/PyRoute/Inputs/ParseStarInput.py +++ b/PyRoute/Inputs/ParseStarInput.py @@ -197,11 +197,15 @@ def _unpack_starline(star, line, sector): else: result, line = ParseStarInput.parser.parse(line) except UnexpectedCharacters: - star.logger.error("Unmatched line: {}".format(line)) - return None, None + result = ParseStarInput._unpack_starline_fallback(line) + if result is None: + star.logger.error("Unmatched line: {}".format(line)) + return None, None except UnexpectedEOF: - star.logger.error("Unmatched line: {}".format(line)) - return None, None + result = ParseStarInput._unpack_starline_fallback(line) + if result is None: + star.logger.error("Unmatched line: {}".format(line)) + return None, None if ParseStarInput.transformer is None: ParseStarInput.transformer = StarlineTransformer(raw=line) else: @@ -220,6 +224,30 @@ def _unpack_starline(star, line, sector): return transformed, is_station + @staticmethod + def _unpack_starline_fallback(line): + matches = ParseStarInput.starline.match(line) + if matches is None: + return + data = list(matches.groups()) + parsed = {'position': data[0], 'name': data[1], 'uwp': data[2], 'trade': data[3]} + parsed['ix'] = data[8] + parsed['ex'] = data[9] + parsed['cx'] = data[10] + parsed['nobles'] = data[11] + parsed['base'] = data[12] + parsed['zone'] = data[13] + parsed['pbg'] = data[14] + parsed['worlds'] = data[15] + parsed['allegiance'] = data[16] + parsed['residual'] = data[17] + + extensions = parsed['ix'] + ' ' + parsed['ex'] + ' ' + parsed['cx'] + + spacer = ' ' + data = [parsed['position'], parsed['name'], parsed['uwp'], parsed['trade'], extensions, parsed['ix'], parsed['ex'], parsed['cx'], spacer, spacer, spacer, parsed['nobles'], parsed['base'], parsed['zone'].upper(), parsed['pbg'], parsed['worlds'], parsed['allegiance'], parsed['residual']] + return data + @staticmethod def check_tl(star, fullmsg=None): if '???-' in str(star.uwp) or star.tl_unknown: From b4944b3214a0fd1d1ea15f7e292e311664cc83f8 Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Sat, 16 Nov 2024 17:39:54 +1000 Subject: [PATCH 2/6] Tweak fallback parsing --- PyRoute/Inputs/ParseStarInput.py | 22 ++++++++++++++++------ Tests/Hypothesis/testStar.py | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/PyRoute/Inputs/ParseStarInput.py b/PyRoute/Inputs/ParseStarInput.py index 1ad1d4a0c..faf2f6c95 100644 --- a/PyRoute/Inputs/ParseStarInput.py +++ b/PyRoute/Inputs/ParseStarInput.py @@ -217,10 +217,13 @@ def _unpack_starline(star, line, sector): ParseStarInput.station_transformer.raw = line ParseStarInput.station_transformer.crankshaft = False - if is_station: - transformed = ParseStarInput.station_transformer.transform(result) + if not isinstance(result, list): + if is_station: + transformed = ParseStarInput.station_transformer.transform(result) + else: + transformed = ParseStarInput.transformer.transform(result) else: - transformed = ParseStarInput.transformer.transform(result) + transformed = result return transformed, is_station @@ -231,9 +234,16 @@ def _unpack_starline_fallback(line): return data = list(matches.groups()) parsed = {'position': data[0], 'name': data[1], 'uwp': data[2], 'trade': data[3]} - parsed['ix'] = data[8] - parsed['ex'] = data[9] - parsed['cx'] = data[10] + raw_extensions = data[4].replace(' ', ' ') + if 2 <= raw_extensions.count(' '): + bitz = raw_extensions.split(' ') + parsed['ix'] = bitz[0] + parsed['ex'] = bitz[1] + parsed['cx'] = bitz[2] + else: + parsed['ix'] = ' ' if data[8] is None else data[8] + parsed['ex'] = ' ' if data[9] is None else data[9] + parsed['cx'] = ' ' if data[10] is None else data[10] parsed['nobles'] = data[11] parsed['base'] = data[12] parsed['zone'] = data[13] diff --git a/Tests/Hypothesis/testStar.py b/Tests/Hypothesis/testStar.py index 5e7c7a112..fbb52752a 100644 --- a/Tests/Hypothesis/testStar.py +++ b/Tests/Hypothesis/testStar.py @@ -116,6 +116,9 @@ def setUp(self) -> None: @example('0101 000000000000000 A0000y0-0 000000000000000 - - 0 000 0000D') @example('0110 000000000000000 ???????-? 000000000000000 H B A 000 0 00') @example('0110 000000000000000 ???????-? 000000000000000 e - A 000 00') + @example('0000 000000000000000 0000000-0 0000000000000 B - A A 000 00') + @example('0000 000000000000000 0000000-0 [00000000000000 - - [0000] - - A 000 00') + @example('0110 000000000000000 ???????-? 0000000000 (000 - (000-0) [0000] - - A 000 0 00') def test_parse_line_to_star(self, s): hyp_line = "Hypothesis input: " + s sector = Sector('# Core', '# 0, 0') From add85f5a37ea29e8ea385e7bb3c310d701544b72 Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Sat, 16 Nov 2024 23:07:07 +1000 Subject: [PATCH 3/6] Fix test blast damage --- Tests/Hypothesis/Inputs/testHypothesisStarlineParser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tests/Hypothesis/Inputs/testHypothesisStarlineParser.py b/Tests/Hypothesis/Inputs/testHypothesisStarlineParser.py index 366e9723a..637198ac6 100644 --- a/Tests/Hypothesis/Inputs/testHypothesisStarlineParser.py +++ b/Tests/Hypothesis/Inputs/testHypothesisStarlineParser.py @@ -81,7 +81,8 @@ def comparison_line(draw): '0000 000000000000000 ???????-? (0 0000000000)A - - A 000 ?0', '2732 sa0)OkeWHEk4Mmb ???????-? (8PvzPS]UbZypt6 - (8Pv-2) [GZks] f * g 88C E7', '0000 000000000000000 ???????-? [00000000000]0] - - A 000 ?0', - '0000 000000000000000 ???????-? (00000000000000 - (000-0) [0000] BB - A 000 ?0' + '0000 000000000000000 ???????-? (00000000000000 - (000-0) [0000] BB - A 000 ?0', + '0000 000000000000000 ???????-? (00000000000)0) - - A 000 00?' ] candidate = draw(from_regex(regex=ParseStarInput.starline, alphabet='0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWYXZ -{}()[]?\'+*')) @@ -320,6 +321,7 @@ class testHypothesisStarlineParser(unittest.TestCase): @example('0001 000000000000000 ???????-? 000000000000000 - - A 000 --0', False) @example('0111 -2Z4ig11RbxW010 0000001-1 wJED9E(E(T (HN6 - (113-0) - - - B 114 00y', False) @example('0000 000000000000000 ???????-? [0000000000]00] - - A 000 ?0', False) + @example('0000 000000000000000 ???????-? (00000000000)0) - - A 000 00?', False) # Weird parsing cases @example('0000 000000000000000 ???????-? (00000000000000 - - 0 000 00?)', 'weird') @example('0000 000000000000000 ???????-? [00000000000000 - - 0 000 00?]', 'weird') From fc8cedd7e59b8390c552c5a8fe02ad8fc25d25eb Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Mon, 18 Nov 2024 23:11:11 +1000 Subject: [PATCH 4/6] Fix corner case in transformer --- PyRoute/Inputs/BaseTransformer.py | 12 ++++++++++++ PyRoute/Inputs/ParseStarInput.py | 4 +++- Tests/Hypothesis/testStar.py | 5 +++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/PyRoute/Inputs/BaseTransformer.py b/PyRoute/Inputs/BaseTransformer.py index fcb676852..e246d485f 100644 --- a/PyRoute/Inputs/BaseTransformer.py +++ b/PyRoute/Inputs/BaseTransformer.py @@ -390,9 +390,21 @@ def trim_raw_string(self, tree): self.raw = self.raw[index:] def _square_up_parsed_zero(self, rawstring, parsed): + from PyRoute.Inputs.ParseStarInput import ParseStarInput bitz = [item for item in rawstring.split(' ') if '' != item] if 3 == len(bitz) and bitz[0] == parsed['nobles'] and bitz[1] == parsed['base'] and bitz[2] == parsed['zone']: return parsed + if 2 == len(bitz) and "" == parsed['zone']: + if 2 < len(bitz[0]): # bitz[0] can only possibly be nobles, so return + return parsed + if 1 < len(bitz[1]): # if bitz[1] is more than one char, it can't be a trade zone, so return + return parsed + non_noble = [item for item in bitz[0] if item not in ParseStarInput.valid_nobles] + if 0 < len(non_noble): # If one or more chars in bitz[0] is not a valid noble call, then we have a base code and trade zone + parsed['zone'] = parsed['base'] + parsed['base'] = parsed['nobles'] + parsed['nobles'] = '' + return parsed if 3 == len(bitz): parsed['nobles'] = bitz[0] parsed['base'] = bitz[1] diff --git a/PyRoute/Inputs/ParseStarInput.py b/PyRoute/Inputs/ParseStarInput.py index faf2f6c95..57aa59085 100644 --- a/PyRoute/Inputs/ParseStarInput.py +++ b/PyRoute/Inputs/ParseStarInput.py @@ -50,6 +50,8 @@ class ParseStarInput: station_parser = None station_transformer = None deep_space = {} + valid_zone = 'arufgbARUFGB-' + valid_nobles = 'BCcDEeFfGH-' @staticmethod def parse_line_into_star_core(star, line, sector, pop_code, ru_calc, fix_pop=False): @@ -84,7 +86,7 @@ def parse_line_into_star_core(star, line, sector, pop_code, ru_calc, fix_pop=Fal if ('' == star.baseCode) or ('-' != star.baseCode and 1 == len(star.baseCode) and not star.baseCode.isalpha()): star.baseCode = '-' star.zone = data[13].strip() - if not star.zone or star.zone not in 'arufgbARUFGB-': + if not star.zone or star.zone not in ParseStarInput.valid_zone: star.zone = '-' star.zone = star.zone.upper() star.ggCount = 0 if (len(data[14]) < 3 or not data[14][2] or data[14][2] in 'X?') else int(data[14][2], 16) diff --git a/Tests/Hypothesis/testStar.py b/Tests/Hypothesis/testStar.py index fbb52752a..85f5b463e 100644 --- a/Tests/Hypothesis/testStar.py +++ b/Tests/Hypothesis/testStar.py @@ -119,6 +119,11 @@ def setUp(self) -> None: @example('0000 000000000000000 0000000-0 0000000000000 B - A A 000 00') @example('0000 000000000000000 0000000-0 [00000000000000 - - [0000] - - A 000 00') @example('0110 000000000000000 ???????-? 0000000000 (000 - (000-0) [0000] - - A 000 0 00') + @example('0101 000000000000000 0000000-0 000000000000000 BB A A 000 0 00') + @example('0101 000000000000000 ???????-? 0000000000000 0 0 B - A 000 00') + @example('0101 000000000000000 ???????-? 000000000000000 {-0} (000-0) - * A 000 00') + @example('0101 000000000000000 ???????-? 000000000000000 {-0} (000-0) - - - A 000 00') + @example("0101 000000000000000 ?000000-0 {3 } (7TG-1) [2IBt] EHEG - 569 9 C5'") def test_parse_line_to_star(self, s): hyp_line = "Hypothesis input: " + s sector = Sector('# Core', '# 0, 0') From f5fa6c30d8739d406416f8721dcce602059e0f0e Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Sun, 24 Nov 2024 00:10:48 +1000 Subject: [PATCH 5/6] Tweak regexen in parsers and ParseStarInput to more closely match UWP regex --- PyRoute/Inputs/ParseStarInput.py | 11 +++++++---- PyRoute/Inputs/StarlineParser.py | 4 ++-- PyRoute/Inputs/StarlineStationParser.py | 4 ++-- PyRoute/SystemData/UWP.py | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/PyRoute/Inputs/ParseStarInput.py b/PyRoute/Inputs/ParseStarInput.py index 57aa59085..e798e1924 100644 --- a/PyRoute/Inputs/ParseStarInput.py +++ b/PyRoute/Inputs/ParseStarInput.py @@ -21,9 +21,9 @@ class ParseStarInput: regex = """ ^([0-3]\d[0-4]\d) + (.{15,}) + -(\w\w\w\w\w\w\w-\w|\?\?\?\?\?\?\?-\?|[\w\?]{7,7}-[\w\?]) + +([A-HXYa-hxy][0-9A-Fa-f]\w\w[0-9A-Fa-f][0-9A-Xa-x][0-9A-Ja-j]-\w|\?\?\?\?\?\?\?-\?|[A-HXYa-hxy\?][0-9A-Fa-f\?][\w\?]{2,2}[0-9A-Fa-f\?][0-9A-Xa-x\?][0-9A-Ja-j\?]-[\w\?]) + (.{15,}) + -((\{ *[+-]?[0-6] ?\}|-) +(\([0-9A-Za-z]{3}[+-]\d\)|- ) +(\[[0-9A-Za-z]{4}\]|- )|( ) ( ) ( )) + +((\{ *[ +-]?[0-6] ?\}|-) +(\([0-9A-Za-z]{3}[+-]\d\)|- ) +(\[[0-9A-Za-z]{4}\]|- )|( ) ( ) ( )) + ([BcCDeEfFGH]{1,5}|-| ) + ([A-Z]{1,3}|-|\*) + ([ARUFGBarufgb]|-| ) + @@ -119,7 +119,10 @@ def parse_line_into_star_core(star, line, sector, pop_code, ru_calc, fix_pop=Fal star.tradeCode.check_world_codes(star, fix_pop=fix_pop) if data[5] and data[5].startswith('{'): - imp = int(data[5][1:-1].strip()) + raw_imp = data[5][1:-1].strip() + imp = 0 + if '' != raw_imp: + imp = int(raw_imp) star.calculate_importance() if imp != star.importance: star.logger.warning( @@ -236,7 +239,7 @@ def _unpack_starline_fallback(line): return data = list(matches.groups()) parsed = {'position': data[0], 'name': data[1], 'uwp': data[2], 'trade': data[3]} - raw_extensions = data[4].replace(' ', ' ') + raw_extensions = data[4].replace(' ', ' ').replace('{ ', '{').replace(' }', '}') if 2 <= raw_extensions.count(' '): bitz = raw_extensions.split(' ') parsed['ix'] = bitz[0] diff --git a/PyRoute/Inputs/StarlineParser.py b/PyRoute/Inputs/StarlineParser.py index 2d33f4876..f5108f42d 100644 --- a/PyRoute/Inputs/StarlineParser.py +++ b/PyRoute/Inputs/StarlineParser.py @@ -20,7 +20,7 @@ class StarlineParser: position: /^([0-3]\d[0-4]\d)/ - starname: /(.{15,}) ([\w\?]{7,7}-[\w\?]) / + starname: /(.{15,}) ([A-HXYa-hxy\?][0-9A-Fa-f\?][\w\?]{2,2}[0-9A-Fa-f\?][0-9A-Xa-x\?][0-9A-Ja-j\?]-[\w\?]) / trade: TRADECODE* TRADECODE: MINOR_DIEBACK | BINARY | POPCODE | MINOR_SOPHONT | OWNED_COLONY | MAJOR_SOPHONT | RESIDUAL | SINGLETON @@ -35,7 +35,7 @@ class StarlineParser: extensions: ix ex cx | /( ) ( ) ( )/ - ix: /\{ *[+-]?[0-6] ?\}|-/ + ix: /\{ *[ +-]?[0-6] ?\}|-/ ex: /\([0-9A-Za-z]{3}[+-]\d\)|-/ cx: /(\[[0-9A-Za-z]{4}[\]\}]|-)/ diff --git a/PyRoute/Inputs/StarlineStationParser.py b/PyRoute/Inputs/StarlineStationParser.py index 0f3a631a7..8583ec122 100644 --- a/PyRoute/Inputs/StarlineStationParser.py +++ b/PyRoute/Inputs/StarlineStationParser.py @@ -20,7 +20,7 @@ class StarlineStationParser: position: /^([0-3]\d[0-4]\d)/ - starname: /(.{15,}) ([\w\?]{7,7}-[\w\?]) / + starname: /(.{15,}) ([A-HXYa-hxy\?][0-9A-Fa-f\?][\w\?]{2,2}[0-9A-Fa-f\?][0-9A-Xa-x\?][0-9A-Ja-j\?]-[\w\?]) / trade: TRADECODE* TRADECODE: MINOR_DIEBACK | BINARY | POPCODE | MINOR_SOPHONT | OWNED_COLONY | MAJOR_SOPHONT | RESIDUAL | SINGLETON @@ -35,7 +35,7 @@ class StarlineStationParser: extensions: ix ex cx | /( ) ( ) ( )/ - ix: /\{ *[+-]?[0-6] ?\}/ + ix: /\{ *[ +-]?[0-6] ?\}/ ex: /\([0-9A-Za-z]{3}[+-]\d\)|-/ cx: /(\[[0-9A-Za-z]{4}[\]\}]|-)/ diff --git a/PyRoute/SystemData/UWP.py b/PyRoute/SystemData/UWP.py index eb272c840..06c0e1600 100644 --- a/PyRoute/SystemData/UWP.py +++ b/PyRoute/SystemData/UWP.py @@ -13,7 +13,7 @@ class UWP(object): # Port code, size, atmo, hydro, pop, gov, law, the all-important hyphen, then TL - match_string = '^([A-HXYa-hxy\?])([0-9A-Fa-f\?])([0-9A-Za-z\?])([0-9A-Za-z\?])([0-9A-Fa-f\?])([0-9A-Za-z\?])([0-9A-Ja-j\?])-([0-9A-Za-z\?])' + match_string = '^([A-HXYa-hxy\?])([0-9A-Fa-f\?])([0-9A-Za-z\?])([0-9A-Za-z\?])([0-9A-Fa-f\?])([0-9A-Xa-x\?])([0-9A-Ja-j\?])-([0-9A-Za-z\?])' match = re.compile(match_string) From 14d5cf0e20da424199af5da17f1f26aa1ff4f9c4 Mon Sep 17 00:00:00 2001 From: Alex Goodwin Date: Sun, 24 Nov 2024 00:33:25 +1000 Subject: [PATCH 6/6] Fix test blast damage --- Tests/Inputs/testStarlineParser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/Inputs/testStarlineParser.py b/Tests/Inputs/testStarlineParser.py index 48a3cffc7..453a29d5e 100644 --- a/Tests/Inputs/testStarlineParser.py +++ b/Tests/Inputs/testStarlineParser.py @@ -161,7 +161,7 @@ def test_parser_screwball_synthetic_starline_5(self): self.assertEqual('', transformed[17], 'Unexpected residual') def test_parser_screwball_synthetic_starline_6(self): - txt = '0000 000000000000000 0000000-0 000000000000000 - - R 000 000000' + txt = '0000 000000000000000 ?000000-0 000000000000000 - - R 000 000000' foo = StarlineParser() result, txt = foo.parse(txt) @@ -171,7 +171,7 @@ def test_parser_screwball_synthetic_starline_6(self): self.assertTrue(isinstance(transformed, list), "Transformed result not list") self.assertEqual('0000', transformed[0], 'Unexpected hex position') self.assertEqual('000000000000000', transformed[1], 'Unexpected name') - self.assertEqual('0000000-0', transformed[2], 'Unexpected UWP') + self.assertEqual('?000000-0', transformed[2], 'Unexpected UWP') self.assertEqual('000000000000000', transformed[3], 'Unexpected trade codes') self.assertEqual('', transformed[4]) self.assertEqual(None, transformed[5]) @@ -261,7 +261,7 @@ def test_parser_screwball_synthetic_starline_9(self): self.assertEqual('', transformed[17], 'Unexpected residual') def test_parser_screwball_synthetic_starline_10(self): - txt = '0000 000000000000000 0000000-0 000000000000000 - - - 000 00' + txt = '0000 000000000000000 ?000000-0 000000000000000 - - - 000 00' foo = StarlineParser() result, txt = foo.parse(txt) @@ -271,7 +271,7 @@ def test_parser_screwball_synthetic_starline_10(self): self.assertTrue(isinstance(transformed, list), "Transformed result not list") self.assertEqual('0000', transformed[0], 'Unexpected hex position') self.assertEqual('000000000000000', transformed[1], 'Unexpected name') - self.assertEqual('0000000-0', transformed[2], 'Unexpected UWP') + self.assertEqual('?000000-0', transformed[2], 'Unexpected UWP') self.assertEqual('000000000000000', transformed[3], 'Unexpected trade codes') self.assertEqual('', transformed[4]) self.assertEqual(None, transformed[5])