From 63963f59f05a75eab20511a03e859768b0804501 Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Wed, 11 Nov 2020 18:21:57 -0500 Subject: [PATCH 01/13] trying to resolve references in external file --- prance/util/resolver.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 3b659c1..5607664 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -110,7 +110,8 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): for _, refstring, item_path in reference_iterator(partial): # Split the reference string into parsed URL and object path ref_url, obj_path = _url.split_url_reference(base_url, refstring) - + # if base_url.path != self.parsed_url.path: + # print("INTERNAL FILE REF") if self._skip_reference(base_url, ref_url): continue @@ -136,7 +137,10 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): full_path = path + item_path # First yield parent - yield full_path, ref_value + if not self.__resolve_types & RESOLVE_INTERNAL and base_url.path != self.parsed_url.path: + yield tuple(obj_path), ref_value + else: + yield full_path, ref_value def _skip_reference(self, base_url, ref_url): """Return whether the URL should not be dereferenced.""" @@ -144,8 +148,11 @@ def _skip_reference(self, base_url, ref_url): return (self.__resolve_types & RESOLVE_HTTP) == 0 elif ref_url.scheme == 'file': # Internal references - if base_url.path == ref_url.path: - return (self.__resolve_types & RESOLVE_INTERNAL) == 0 + # Recursive Object + if base_url.fragment == ref_url.fragment: + return True + # if base_url.path == ref_url.path: + # return (self.__resolve_types & RESOLVE_INTERNAL) == 0 # Local files return (self.__resolve_types & RESOLVE_FILES) == 0 else: @@ -168,12 +175,12 @@ def _dereference(self, ref_url, obj_path, recursions): # In order to start dereferencing anything in the referenced URL, we have # to read and parse it, of course. contents = _url.fetch_url(ref_url, self.__reference_cache, self.__encoding) - # In this inner parser's specification, we can now look for the referenced # object. value = contents if len(obj_path) != 0: from prance.util.path import path_get + print("obj_path", obj_path) try: value = path_get(value, obj_path) except (KeyError, IndexError, TypeError) as ex: @@ -186,8 +193,8 @@ def _dereference(self, ref_url, obj_path, recursions): # Now resolve partial specs value = self._resolve_partial(ref_url, value, recursions) - # That's it! + print(value) return value def _resolve_partial(self, base_url, partial, recursions): @@ -204,7 +211,6 @@ def _resolve_partial(self, base_url, partial, recursions): # sorting paths by path length. changes = dict(tuple(self._dereferencing_iterator(base_url, partial, (), recursions))) - paths = sorted(changes.keys(), key = len) # With the paths sorted, set them to the resolved values. @@ -214,6 +220,7 @@ def _resolve_partial(self, base_url, partial, recursions): if len(path) == 0: partial = value else: + print("partial_path", path) path_set(partial, list(path), value, create = True) return partial From a52c111d8dc2dcd8cf2473d980d4fdd216185916 Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Thu, 12 Nov 2020 15:02:45 -0500 Subject: [PATCH 02/13] super hackey commit; works on our specific schema; does not work for nested objects --- prance/__init__.py | 2 +- prance/util/iterators.py | 1 + prance/util/resolver.py | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/prance/__init__.py b/prance/__init__.py index 505cb34..d4008cb 100644 --- a/prance/__init__.py +++ b/prance/__init__.py @@ -293,6 +293,6 @@ def _validate(self): ) resolver.resolve_references() self.specification = resolver.specs - + self.specification["components"]["schemas"].update(resolver.soft_derefence_objs) # Now validate - the BaseParser knows the specifics BaseParser._validate(self) diff --git a/prance/util/iterators.py b/prance/util/iterators.py index bb99acb..26fd625 100644 --- a/prance/util/iterators.py +++ b/prance/util/iterators.py @@ -85,4 +85,5 @@ def reference_iterator(specs, path = ()): continue key = item_path[-1] if key == '$ref': + # print("key", key, "item", item, "item_path", item_path[:-1]) yield key, item, item_path[:-1] diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 5607664..a28ee67 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -90,6 +90,7 @@ def __init__(self, specs, url = None, **options): self.__resolve_types = options.get('resolve_types', RESOLVE_ALL) self.__encoding = options.get('encoding', None) + self.soft_derefence_objs = {} def resolve_references(self): """Resolve JSON pointers/references in the spec.""" @@ -110,6 +111,7 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): for _, refstring, item_path in reference_iterator(partial): # Split the reference string into parsed URL and object path ref_url, obj_path = _url.split_url_reference(base_url, refstring) + # print("ref_url", ref_url.path, "obj_path", obj_path, "ref_string", refstring) # if base_url.path != self.parsed_url.path: # print("INTERNAL FILE REF") if self._skip_reference(base_url, ref_url): @@ -138,19 +140,26 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): # First yield parent if not self.__resolve_types & RESOLVE_INTERNAL and base_url.path != self.parsed_url.path: - yield tuple(obj_path), ref_value + # dref_url = ref_url.path.split("/")[-1]+"/"+"/".join(obj_path[1:]) + dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(obj_path[1:]) + # base_url.path.split("/")[-1]+"#"+"/".join(obj_path) + self.soft_derefence_objs[dref_url] = ref_value + # print("base_url", "#/"+dref_url, full_path, type(ref_value)) + yield full_path, {"$ref": "#/components/schemas/"+dref_url} + # yield tuple(obj_path), ref_value else: yield full_path, ref_value def _skip_reference(self, base_url, ref_url): """Return whether the URL should not be dereferenced.""" + # print("REF URL SCHEME", ref_url.scheme) if ref_url.scheme.startswith('http'): return (self.__resolve_types & RESOLVE_HTTP) == 0 elif ref_url.scheme == 'file': # Internal references # Recursive Object - if base_url.fragment == ref_url.fragment: - return True + # if base_url.fragment == ref_url.fragment: + # return True # if base_url.path == ref_url.path: # return (self.__resolve_types & RESOLVE_INTERNAL) == 0 # Local files @@ -180,7 +189,7 @@ def _dereference(self, ref_url, obj_path, recursions): value = contents if len(obj_path) != 0: from prance.util.path import path_get - print("obj_path", obj_path) + # print("obj_path", obj_path) try: value = path_get(value, obj_path) except (KeyError, IndexError, TypeError) as ex: @@ -194,7 +203,7 @@ def _dereference(self, ref_url, obj_path, recursions): # Now resolve partial specs value = self._resolve_partial(ref_url, value, recursions) # That's it! - print(value) + # print(value) return value def _resolve_partial(self, base_url, partial, recursions): @@ -220,7 +229,7 @@ def _resolve_partial(self, base_url, partial, recursions): if len(path) == 0: partial = value else: - print("partial_path", path) + # print("partial_path", path) path_set(partial, list(path), value, create = True) return partial From 020f99570ab77dc9f9634920a363a59b65f13ad7 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Fri, 13 Nov 2020 10:53:09 +0100 Subject: [PATCH 03/13] cleanup --- prance/util/iterators.py | 1 - prance/util/resolver.py | 15 ++++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/prance/util/iterators.py b/prance/util/iterators.py index 26fd625..bb99acb 100644 --- a/prance/util/iterators.py +++ b/prance/util/iterators.py @@ -85,5 +85,4 @@ def reference_iterator(specs, path = ()): continue key = item_path[-1] if key == '$ref': - # print("key", key, "item", item, "item_path", item_path[:-1]) yield key, item, item_path[:-1] diff --git a/prance/util/resolver.py b/prance/util/resolver.py index a28ee67..83783a3 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -111,9 +111,7 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): for _, refstring, item_path in reference_iterator(partial): # Split the reference string into parsed URL and object path ref_url, obj_path = _url.split_url_reference(base_url, refstring) - # print("ref_url", ref_url.path, "obj_path", obj_path, "ref_string", refstring) - # if base_url.path != self.parsed_url.path: - # print("INTERNAL FILE REF") + if self._skip_reference(base_url, ref_url): continue @@ -140,19 +138,14 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): # First yield parent if not self.__resolve_types & RESOLVE_INTERNAL and base_url.path != self.parsed_url.path: - # dref_url = ref_url.path.split("/")[-1]+"/"+"/".join(obj_path[1:]) dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(obj_path[1:]) - # base_url.path.split("/")[-1]+"#"+"/".join(obj_path) self.soft_derefence_objs[dref_url] = ref_value - # print("base_url", "#/"+dref_url, full_path, type(ref_value)) yield full_path, {"$ref": "#/components/schemas/"+dref_url} - # yield tuple(obj_path), ref_value else: yield full_path, ref_value def _skip_reference(self, base_url, ref_url): """Return whether the URL should not be dereferenced.""" - # print("REF URL SCHEME", ref_url.scheme) if ref_url.scheme.startswith('http'): return (self.__resolve_types & RESOLVE_HTTP) == 0 elif ref_url.scheme == 'file': @@ -184,12 +177,12 @@ def _dereference(self, ref_url, obj_path, recursions): # In order to start dereferencing anything in the referenced URL, we have # to read and parse it, of course. contents = _url.fetch_url(ref_url, self.__reference_cache, self.__encoding) + # In this inner parser's specification, we can now look for the referenced # object. value = contents if len(obj_path) != 0: from prance.util.path import path_get - # print("obj_path", obj_path) try: value = path_get(value, obj_path) except (KeyError, IndexError, TypeError) as ex: @@ -202,8 +195,8 @@ def _dereference(self, ref_url, obj_path, recursions): # Now resolve partial specs value = self._resolve_partial(ref_url, value, recursions) + # That's it! - # print(value) return value def _resolve_partial(self, base_url, partial, recursions): @@ -220,6 +213,7 @@ def _resolve_partial(self, base_url, partial, recursions): # sorting paths by path length. changes = dict(tuple(self._dereferencing_iterator(base_url, partial, (), recursions))) + paths = sorted(changes.keys(), key = len) # With the paths sorted, set them to the resolved values. @@ -229,7 +223,6 @@ def _resolve_partial(self, base_url, partial, recursions): if len(path) == 0: partial = value else: - # print("partial_path", path) path_set(partial, list(path), value, create = True) return partial From 10016c25211f690417e7b76704147c2721d80b2b Mon Sep 17 00:00:00 2001 From: Glutexo Date: Fri, 13 Nov 2020 11:48:58 +0100 Subject: [PATCH 04/13] Add SOFT/HARD switch. --- prance/__init__.py | 2 +- prance/util/resolver.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/prance/__init__.py b/prance/__init__.py index d4008cb..cef4ebe 100644 --- a/prance/__init__.py +++ b/prance/__init__.py @@ -282,7 +282,7 @@ def _validate(self): # We therefore use our own resolver first, and validate later. from .util.resolver import RefResolver forward_arg_names = ('encoding', 'recursion_limit', - 'recursion_limit_handler', 'resolve_types') + 'recursion_limit_handler', 'resolve_types', 'resolve_method') forward_args = { k: v for (k, v) in self.options.items() if k in forward_arg_names } diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 83783a3..ebc8800 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -15,6 +15,11 @@ #: Resolve references to local files. RESOLVE_FILES = 2 ** 3 +#: Copy the schema changing the reference. +RESOLVE_SOFT = 0 +#: Replace the reference with inlined schema. +RESOLVE_HARD = 1 + #: Default, resole all references. RESOLVE_ALL = RESOLVE_INTERNAL | RESOLVE_HTTP | RESOLVE_FILES @@ -89,6 +94,7 @@ def __init__(self, specs, url = None, **options): self.parsed_url = self._url_key = None self.__resolve_types = options.get('resolve_types', RESOLVE_ALL) + self.__resolve_method = options.get('resolve_method', RESOLVE_HARD) self.__encoding = options.get('encoding', None) self.soft_derefence_objs = {} @@ -137,7 +143,11 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): full_path = path + item_path # First yield parent - if not self.__resolve_types & RESOLVE_INTERNAL and base_url.path != self.parsed_url.path: + if ( + self.__resolve_method == RESOLVE_SOFT and + not self.__resolve_types & RESOLVE_INTERNAL and + base_url.path != self.parsed_url.path + ): dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(obj_path[1:]) self.soft_derefence_objs[dref_url] = ref_value yield full_path, {"$ref": "#/components/schemas/"+dref_url} @@ -153,8 +163,8 @@ def _skip_reference(self, base_url, ref_url): # Recursive Object # if base_url.fragment == ref_url.fragment: # return True - # if base_url.path == ref_url.path: - # return (self.__resolve_types & RESOLVE_INTERNAL) == 0 + if base_url.path == ref_url.path: + return (self.__resolve_types & RESOLVE_INTERNAL) == 0 # Local files return (self.__resolve_types & RESOLVE_FILES) == 0 else: From 59e877c6933b008f44424e947e4311f144a3a68f Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Tue, 17 Nov 2020 15:53:02 -0500 Subject: [PATCH 05/13] update soft ref objects internally --- prance/__init__.py | 1 - prance/util/resolver.py | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/prance/__init__.py b/prance/__init__.py index cef4ebe..15627d3 100644 --- a/prance/__init__.py +++ b/prance/__init__.py @@ -293,6 +293,5 @@ def _validate(self): ) resolver.resolve_references() self.specification = resolver.specs - self.specification["components"]["schemas"].update(resolver.soft_derefence_objs) # Now validate - the BaseParser knows the specifics BaseParser._validate(self) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index ebc8800..f7dc632 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -96,11 +96,13 @@ def __init__(self, specs, url = None, **options): self.__resolve_types = options.get('resolve_types', RESOLVE_ALL) self.__resolve_method = options.get('resolve_method', RESOLVE_HARD) self.__encoding = options.get('encoding', None) - self.soft_derefence_objs = {} + self.__soft_derefence_objs = {} def resolve_references(self): """Resolve JSON pointers/references in the spec.""" self.specs = self._resolve_partial(self.parsed_url, self.specs, ()) + + if self.__soft_derefence_objs: self.specs["components"]["schemas"].update(self.__soft_derefence_objs) def _dereferencing_iterator(self, base_url, partial, path, recursions): """ @@ -145,11 +147,10 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): # First yield parent if ( self.__resolve_method == RESOLVE_SOFT and - not self.__resolve_types & RESOLVE_INTERNAL and base_url.path != self.parsed_url.path ): dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(obj_path[1:]) - self.soft_derefence_objs[dref_url] = ref_value + self.__soft_derefence_objs[dref_url] = ref_value yield full_path, {"$ref": "#/components/schemas/"+dref_url} else: yield full_path, ref_value @@ -160,9 +161,6 @@ def _skip_reference(self, base_url, ref_url): return (self.__resolve_types & RESOLVE_HTTP) == 0 elif ref_url.scheme == 'file': # Internal references - # Recursive Object - # if base_url.fragment == ref_url.fragment: - # return True if base_url.path == ref_url.path: return (self.__resolve_types & RESOLVE_INTERNAL) == 0 # Local files From 63909e8e367e5e4108b9745f8ee36470e257fbca Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Wed, 18 Nov 2020 16:55:48 -0500 Subject: [PATCH 06/13] dereferencing file schema in a different object --- prance/util/resolver.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index f7dc632..21e4ebc 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -96,13 +96,13 @@ def __init__(self, specs, url = None, **options): self.__resolve_types = options.get('resolve_types', RESOLVE_ALL) self.__resolve_method = options.get('resolve_method', RESOLVE_HARD) self.__encoding = options.get('encoding', None) - self.__soft_derefence_objs = {} + self.__soft_dereference_objs = {} def resolve_references(self): """Resolve JSON pointers/references in the spec.""" self.specs = self._resolve_partial(self.parsed_url, self.specs, ()) - if self.__soft_derefence_objs: self.specs["components"]["schemas"].update(self.__soft_derefence_objs) + if self.__soft_dereference_objs: self.specs["components"]["schemas"].update(self.__soft_dereference_objs) def _dereferencing_iterator(self, base_url, partial, path, recursions): """ @@ -145,15 +145,23 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): full_path = path + item_path # First yield parent + # if soft resolve is enabled, add collect those in soft_dereference_objs if ( - self.__resolve_method == RESOLVE_SOFT and - base_url.path != self.parsed_url.path + (self.__resolve_method == RESOLVE_SOFT and + base_url.path != self.parsed_url.path) or ( + self.__resolve_method == RESOLVE_SOFT and + base_url.path != ref_url.path + ) ): - dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(obj_path[1:]) - self.__soft_derefence_objs[dref_url] = ref_value - yield full_path, {"$ref": "#/components/schemas/"+dref_url} + url = self._collect_soft_refs(ref_url, obj_path, ref_value) + yield full_path, {"$ref": "#/components/schemas/"+url} else: yield full_path, ref_value + + def _collect_soft_refs(self, ref_url, item_path, value): + dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(item_path[1:]) + self.__soft_dereference_objs[dref_url] = value + return dref_url def _skip_reference(self, base_url, ref_url): """Return whether the URL should not be dereferenced.""" From 0740b33c83afb90e6208317bdd1802b9a6b9c046 Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Thu, 19 Nov 2020 15:44:25 -0500 Subject: [PATCH 07/13] create components/schemas if not available, added comments --- prance/__init__.py | 1 + prance/util/resolver.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/prance/__init__.py b/prance/__init__.py index 15627d3..2ea41d9 100644 --- a/prance/__init__.py +++ b/prance/__init__.py @@ -293,5 +293,6 @@ def _validate(self): ) resolver.resolve_references() self.specification = resolver.specs + # Now validate - the BaseParser knows the specifics BaseParser._validate(self) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 21e4ebc..3b51492 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -102,7 +102,12 @@ def resolve_references(self): """Resolve JSON pointers/references in the spec.""" self.specs = self._resolve_partial(self.parsed_url, self.specs, ()) - if self.__soft_dereference_objs: self.specs["components"]["schemas"].update(self.__soft_dereference_objs) + # If there are any objects collected when using RESOLVE_SOFT, add them to components/schemas + if self.__soft_dereference_objs: + if "components" not in self.specs: self.specs["components"] = dict() + if "schemas" not in self.specs["components"]: self.specs["components"].update({"schemas":{}}) + + self.specs["components"]["schemas"].update(self.__soft_dereference_objs) def _dereferencing_iterator(self, base_url, partial, path, recursions): """ @@ -145,7 +150,6 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): full_path = path + item_path # First yield parent - # if soft resolve is enabled, add collect those in soft_dereference_objs if ( (self.__resolve_method == RESOLVE_SOFT and base_url.path != self.parsed_url.path) or ( @@ -153,12 +157,19 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): base_url.path != ref_url.path ) ): + # If RESOLVE_SOFT is enabled and the reference is an external reference, e.g. a file ref + # or, if RESOLVE_SOFT is enabled and the reference is in an external object + # collect the dereferenced object to add them later in components/schemas url = self._collect_soft_refs(ref_url, obj_path, ref_value) yield full_path, {"$ref": "#/components/schemas/"+url} else: yield full_path, ref_value def _collect_soft_refs(self, ref_url, item_path, value): + """ + Returns a portion of the dereferenced url for RESOLVE_SOFT mode. + format - ref-url_obj-path + """ dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(item_path[1:]) self.__soft_dereference_objs[dref_url] = value return dref_url From 342f007fdc03e8f8ba8b7f1bbc694423e86df8ec Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Fri, 20 Nov 2020 11:55:40 -0500 Subject: [PATCH 08/13] simplifying condition and renaming to TRANSLATE EXTERNAL --- prance/util/resolver.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 3b51492..2948aeb 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -16,9 +16,9 @@ RESOLVE_FILES = 2 ** 3 #: Copy the schema changing the reference. -RESOLVE_SOFT = 0 +TRANSLATE_EXTERNAL = 0 #: Replace the reference with inlined schema. -RESOLVE_HARD = 1 +TRANSLATE_DEFAULT = 1 #: Default, resole all references. RESOLVE_ALL = RESOLVE_INTERNAL | RESOLVE_HTTP | RESOLVE_FILES @@ -94,7 +94,7 @@ def __init__(self, specs, url = None, **options): self.parsed_url = self._url_key = None self.__resolve_types = options.get('resolve_types', RESOLVE_ALL) - self.__resolve_method = options.get('resolve_method', RESOLVE_HARD) + self.__resolve_method = options.get('resolve_method', TRANSLATE_DEFAULT) self.__encoding = options.get('encoding', None) self.__soft_dereference_objs = {} @@ -102,7 +102,7 @@ def resolve_references(self): """Resolve JSON pointers/references in the spec.""" self.specs = self._resolve_partial(self.parsed_url, self.specs, ()) - # If there are any objects collected when using RESOLVE_SOFT, add them to components/schemas + # If there are any objects collected when using TRANSLATE_EXTERNAL, add them to components/schemas if self.__soft_dereference_objs: if "components" not in self.specs: self.specs["components"] = dict() if "schemas" not in self.specs["components"]: self.specs["components"].update({"schemas":{}}) @@ -124,8 +124,8 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): for _, refstring, item_path in reference_iterator(partial): # Split the reference string into parsed URL and object path ref_url, obj_path = _url.split_url_reference(base_url, refstring) - - if self._skip_reference(base_url, ref_url): + translate = self.__resolve_method == TRANSLATE_EXTERNAL and self.parsed_url.path != ref_url.path + if not translate and self._skip_reference(base_url, ref_url): continue # The reference path is the url resource and object path @@ -150,16 +150,7 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): full_path = path + item_path # First yield parent - if ( - (self.__resolve_method == RESOLVE_SOFT and - base_url.path != self.parsed_url.path) or ( - self.__resolve_method == RESOLVE_SOFT and - base_url.path != ref_url.path - ) - ): - # If RESOLVE_SOFT is enabled and the reference is an external reference, e.g. a file ref - # or, if RESOLVE_SOFT is enabled and the reference is in an external object - # collect the dereferenced object to add them later in components/schemas + if translate: url = self._collect_soft_refs(ref_url, obj_path, ref_value) yield full_path, {"$ref": "#/components/schemas/"+url} else: @@ -167,7 +158,7 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): def _collect_soft_refs(self, ref_url, item_path, value): """ - Returns a portion of the dereferenced url for RESOLVE_SOFT mode. + Returns a portion of the dereferenced url for TRANSLATE_EXTERNAL mode. format - ref-url_obj-path """ dref_url = ref_url.path.split("/")[-1]+"_"+"_".join(item_path[1:]) From 1f5bd992160afed6c46da071627f16f961397b9f Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Fri, 20 Nov 2020 18:49:40 -0500 Subject: [PATCH 09/13] adding tests --- tests/test_util_resolver.py | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/test_util_resolver.py b/tests/test_util_resolver.py index 35d7adc..cc27db8 100644 --- a/tests/test_util_resolver.py +++ b/tests/test_util_resolver.py @@ -494,3 +494,68 @@ def test_issue_78_resolve_internal_bug(): assert 'application/json' in val # Internal reference within file is NOT resolved assert '$ref' in val['application/json'] + + + +@pytest.mark.skipif(none_of('openapi-spec-validator'), reason='Missing backends') +def test_issue_77_translate_external(): + specs = '' + with open('tests/specs/issue_78/openapi.json', 'r') as fh: + specs = fh.read() + + from prance.util import formats + specs = formats.parse_spec(specs, 'openapi.json') + + res = resolver.RefResolver(specs, + fs.abspath('openapi.json'), + resolve_types = resolver.RESOLVE_FILES, + resolve_method= resolver.TRANSLATE_EXTERNAL + ) + res.resolve_references() + + from prance.util.path import path_get + val = path_get(res.specs, ('components', 'schemas', '_schemas.json_Body')) + + # Reference to file is translated in components/schemas + assert 'content' in val + assert 'application/json' in val['content'] + + # Reference url is updated + val = path_get(res.specs, ('paths', '/endpoint', 'post', 'requestBody', '$ref')) + assert val == '#/components/schemas/_schemas.json_Body' + + + +@pytest.mark.skipif(none_of('openapi-spec-validator'), reason='Missing backends') +def test_issue_77_translate_external_refs_internal(): + specs = '' + with open('tests/specs/issue_78/openapi.json', 'r') as fh: + specs = fh.read() + + from prance.util import formats + specs = formats.parse_spec(specs, 'openapi.json') + + res = resolver.RefResolver(specs, + fs.abspath('openapi.json'), + resolve_types = resolver.RESOLVE_FILES | resolver.RESOLVE_INTERNAL, + resolve_method= resolver.TRANSLATE_EXTERNAL + ) + res.resolve_references() + + from prance.util.path import path_get + val = path_get(res.specs, ('components', 'schemas', '_schemas.json_Body')) + + # Reference to file is translated in components/schemas + assert 'content' in val + assert 'application/json' in val['content'] + + # Internal Reference links updated + assert '#/components/schemas/_schemas.json_Something' == val['content']['application/json']['$ref'] + + # Internal references is copied to componnents/schemas seperately + val = path_get(res.specs, ('components', 'schemas', '_schemas.json_Something')) + assert 'type' in val + + # File reference url is updated as well + val = path_get(res.specs, ('paths', '/endpoint', 'post', 'requestBody', '$ref')) + assert val == '#/components/schemas/_schemas.json_Body' From 07d42c00c9bcc6e868def842b0fc47892eb1eae1 Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Fri, 20 Nov 2020 18:56:34 -0500 Subject: [PATCH 10/13] adding param info --- prance/util/resolver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index 2948aeb..f6fa621 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -71,6 +71,9 @@ def __init__(self, specs, url = None, **options): detect_encoding is used to determine the encoding. :param int resolve_types: [optional] Specify which types of references to resolve. Defaults to RESOLVE_ALL. + :param int resolve_method: [optional] Specify whether to translate external + references in components/schemas or dereference in place. Defaults + to TRANSLATE_DEFAULT. """ import copy self.specs = copy.deepcopy(specs) From a9ca0809616cc4fec3f1db1dd3588ef9d45c224a Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Sun, 22 Nov 2020 14:52:59 -0500 Subject: [PATCH 11/13] removing extra indentation --- prance/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prance/__init__.py b/prance/__init__.py index 2ea41d9..6a1caf9 100644 --- a/prance/__init__.py +++ b/prance/__init__.py @@ -293,6 +293,6 @@ def _validate(self): ) resolver.resolve_references() self.specification = resolver.specs - + # Now validate - the BaseParser knows the specifics BaseParser._validate(self) From d005013246eef613f57de3e438d6f7b85d37aa4c Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Fri, 4 Dec 2020 09:03:14 -0500 Subject: [PATCH 12/13] don't resolve internal refs when RESOLVE_INTERNAL is false translate and skip_references() are independent entities. They should not belong in the same conditional statement. :man_facepalming: --- prance/util/resolver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prance/util/resolver.py b/prance/util/resolver.py index f6fa621..0dcc440 100644 --- a/prance/util/resolver.py +++ b/prance/util/resolver.py @@ -127,8 +127,10 @@ def _dereferencing_iterator(self, base_url, partial, path, recursions): for _, refstring, item_path in reference_iterator(partial): # Split the reference string into parsed URL and object path ref_url, obj_path = _url.split_url_reference(base_url, refstring) + translate = self.__resolve_method == TRANSLATE_EXTERNAL and self.parsed_url.path != ref_url.path - if not translate and self._skip_reference(base_url, ref_url): + + if self._skip_reference(base_url, ref_url): continue # The reference path is the url resource and object path From 05583227f5ea1819229d7ad30e963a4a1546fffa Mon Sep 17 00:00:00 2001 From: tahmidefaz Date: Fri, 4 Dec 2020 09:08:20 -0500 Subject: [PATCH 13/13] test --- tests/test_util_resolver.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_util_resolver.py b/tests/test_util_resolver.py index cc27db8..d2b4c37 100644 --- a/tests/test_util_resolver.py +++ b/tests/test_util_resolver.py @@ -559,3 +559,29 @@ def test_issue_77_translate_external_refs_internal(): # File reference url is updated as well val = path_get(res.specs, ('paths', '/endpoint', 'post', 'requestBody', '$ref')) assert val == '#/components/schemas/_schemas.json_Body' + + +@pytest.mark.skipif(none_of('openapi-spec-validator'), reason='Missing backends') +def test_issue_77_internal_refs_unresolved(): + specs = '' + with open('tests/specs/issue_78/openapi.json', 'r') as fh: + specs = fh.read() + + from prance.util import formats + specs = formats.parse_spec(specs, 'openapi.json') + + res = resolver.RefResolver(specs, + fs.abspath('openapi.json'), + resolve_types = resolver.RESOLVE_FILES, + resolve_method= resolver.TRANSLATE_EXTERNAL + ) + res.resolve_references() + + from prance.util.path import path_get + val = path_get(res.specs, ('components', 'schemas')) + + # File reference resolved + assert '_schemas.json_Body' in val + + # Internal file reference not resolved + assert '_schemas.json_Something' not in val