diff --git a/dash-pipeline/SAI/sai_api_gen.py b/dash-pipeline/SAI/sai_api_gen.py index af8098b48..ceb574d42 100755 --- a/dash-pipeline/SAI/sai_api_gen.py +++ b/dash-pipeline/SAI/sai_api_gen.py @@ -68,7 +68,8 @@ print("Outputting new SAI spec to " + sai_spec_dir) yaml_inc_ctor.autoload = False new_sai_spec = dash_sai_exts.to_sai() - new_sai_spec.serialize(sai_spec_dir) + sai_spec.merge(new_sai_spec) + sai_spec.serialize(sai_spec_dir) # Generate and update all SAI files SAIGenerator(dash_sai_exts).generate() diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api.py b/dash-pipeline/SAI/utils/sai_spec/sai_api.py index fec9001fc..19f10f405 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api.py @@ -4,6 +4,7 @@ from .sai_enum import SaiEnum from .sai_struct import SaiStruct from .sai_api_p4_meta import SaiApiP4Meta +from . import sai_spec_utils class SaiApi(SaiCommon): @@ -19,3 +20,16 @@ def __init__(self, name: str, description: str, is_object: bool = False): self.attributes: List[SaiAttribute] = [] self.stats: List[SaiAttribute] = [] self.p4_meta: SaiApiP4Meta = SaiApiP4Meta() + + def merge(self, other: "SaiCommon"): + super().merge(other) + + self.is_object = other.is_object + sai_spec_utils.merge_sai_common_lists(self.enums, other.enums) + sai_spec_utils.merge_sai_common_lists(self.structs, other.structs) + sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes) + sai_spec_utils.merge_sai_common_lists(self.stats, other.stats) + + # The P4 meta can be merged by replacing the old one, since it doesn't affect the ABI, + # and the new one is always more up-to-date. + self.p4_meta = other.p4_meta diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py index 3ebd9e38a..d3324d88c 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py @@ -1,5 +1,6 @@ from typing import List from .sai_attribute import SaiAttribute +from . import sai_spec_utils class SaiApiExtension: @@ -12,3 +13,7 @@ class SaiApiExtension: def __init__(self): self.attributes: List[SaiAttribute] = [] self.stats: List[SaiAttribute] = [] + + def merge(self, other: "SaiApiExtension"): + sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes) + sai_spec_utils.merge_sai_common_lists(self.stats, other.stats) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py index 25f0b7b38..b960a9f99 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py @@ -1,6 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_api import SaiApi +from . import sai_spec_utils class SaiApiGroup(SaiCommon): @@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon): def __init__(self, name: str, description: str): super().__init__(name, description) self.sai_apis: List[SaiApi] = [] + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis) + + def deprecate(self) -> bool: + """ + Deprecate API group. + + When deprecating the API group, we can safely remove it from the list as the + net effect is the same as keeping it: + - The old API type, object type and object entries will not be changed. + - The SAI headers will not be changed, because their API groups are present. + - The DASH libsai will not be generated anymore, but it is ok, since we will not + use them in the BMv2 anyway. + """ + return True diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py index bdb680f0a..88b0033ff 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py @@ -31,3 +31,11 @@ def __init__( self.allow_null = allow_null self.valid_only = valid_only self.deprecated = deprecated + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.__dict__.update(other.__dict__) + + def deprecate(self) -> bool: + self.deprecated = True + return False \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_common.py b/dash-pipeline/SAI/utils/sai_spec/sai_common.py index 7f1a21332..c6ee8e8cd 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_common.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_common.py @@ -6,3 +6,21 @@ class SaiCommon: def __init__(self, name: str, description: str): self.name: str = name self.description: str = description + + def merge(self, other: "SaiCommon"): + """ + Merge the other SaiCommon object into this object. + """ + if not isinstance(other, type(self)): + raise TypeError(f"Cannot merge {type(self)} with {type(other)}") + + self.description = other.description + + def deprecate(self) -> bool: + """ + Deprecate this object. + + If the value doesn't support deprecation marking, we don't do anything + but return False to keep it in the list. + """ + return False diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py index bae1cff4c..621b80427 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py @@ -1,6 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_enum_member import SaiEnumMember +from . import sai_spec_utils class SaiEnum(SaiCommon): @@ -11,3 +12,7 @@ class SaiEnum(SaiCommon): def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []): super().__init__(name, description) self.members: List[SaiEnumMember] = members + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.members, other.members) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py index f8cfb6bd2..fe4a458c8 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py @@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon): def __init__(self, name: str, description: str, value: str): super().__init__(name, description) self.value: str = value + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.value = other.value \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py index 74cd6ee33..e82faee03 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py @@ -6,6 +6,7 @@ from .sai_api_group import SaiApiGroup from .sai_api_extension import SaiApiExtension from .sai_struct_entry import SaiStructEntry +from . import sai_spec_utils class SaiSpec: @@ -24,12 +25,16 @@ def __init__(self): def serialize(self, spec_dir: str): yaml_inc_files = [] for api_group in self.api_groups: - sai_api_group_spec_file_path = os.path.join(spec_dir, api_group.name + ".yaml") + sai_api_group_spec_file_path = os.path.join( + spec_dir, api_group.name + ".yaml" + ) with open(sai_api_group_spec_file_path, "w") as f: f.write(yaml.dump(api_group, indent=2, sort_keys=False)) - - yaml_inc_files.append(yaml_include.Data(urlpath=sai_api_group_spec_file_path)) + + yaml_inc_files.append( + yaml_include.Data(urlpath=sai_api_group_spec_file_path) + ) api_groups = self.api_groups self.api_groups = yaml_inc_files @@ -43,3 +48,21 @@ def serialize(self, spec_dir: str): def deserialize(spec_dir: str): with open(os.path.join(spec_dir, "sai_spec.yaml")) as f: return yaml.unsafe_load(f) + + def merge(self, other: "SaiSpec"): + sai_spec_utils.merge_sai_value_lists( + self.api_types, other.api_types, lambda x: x + ) + sai_spec_utils.merge_sai_value_lists( + self.object_types, other.object_types, lambda x: x + ) + sai_spec_utils.merge_sai_common_lists(self.object_entries, other.object_entries) + + # The global enums are generated from the P4 enum types, so we can respect whatever in the + # new spec and simply replace them, because: + # - It doesn't matter if the order of enum itself changes. + # - We cannot move the enum members as we want, as their order changes their values. + self.enums = other.enums + + self.port_extenstion.merge(other.port_extenstion) + sai_spec_utils.merge_sai_common_lists(self.api_groups, other.api_groups) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py new file mode 100644 index 000000000..747d19933 --- /dev/null +++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py @@ -0,0 +1,50 @@ +from typing import Any, List, Callable +from .sai_common import SaiCommon + + +def merge_sai_value_lists( + target: List[Any], + source: List[Any], + get_key: Callable[[Any], str], + on_conflict: Callable[[Any, Any], None] = lambda x, y: x, + on_deprecate: Callable[[Any], bool] = lambda x: False +) -> None: + """ + Merge 2 SAI value lists from source list into target. + + Since we could not remove the old value or change the order of old values, the merge + is done as below: + - Any new values will be added in the end of the list. + - Any values that collapse with existing values will invoke on_conflict callback to resolve. + - Any values that needs to be removed will invoke on_deprecate function to deprecate. By default, + it will not be removed from the old list. + """ + target_dict = {get_key(item): item for item in target} + + source_keys = set() + for source_item in source: + source_key = get_key(source_item) + source_keys.add(source_key) + + if source_key in target_dict: + target_item = target_dict[source_key] + on_conflict(target_item, source_item) + else: + target.append(source_item) + target_dict[source_key] = source_item + + # Remove all items in target, if its key doesn't exist in source_keys and on_deprecate returns True. + target[:] = [item for item in target if get_key(item) in source_keys or not on_deprecate(item)] + + +def merge_sai_common_lists( + target: List[SaiCommon], + source: List[SaiCommon], +) -> None: + merge_sai_value_lists( + target, + source, + get_key=lambda x: x.name, + on_conflict=lambda x, y: x.merge(y), + on_deprecate=lambda x: x.deprecate(), + ) diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py index b98cf3a27..025d1ecb4 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py @@ -1,7 +1,7 @@ from typing import List from .sai_common import SaiCommon from .sai_struct_entry import SaiStructEntry - +from . import sai_spec_utils class SaiStruct(SaiCommon): """ @@ -11,3 +11,7 @@ class SaiStruct(SaiCommon): def __init__(self, name: str, description: str, members: List[SaiStructEntry] = []): super().__init__(name, description) self.members: List[SaiStructEntry] = members + + def merge(self, other: "SaiCommon"): + super().merge(other) + sai_spec_utils.merge_sai_common_lists(self.members, other.members) \ No newline at end of file diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py index 1f4e6ea06..ba3e42f99 100644 --- a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py +++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py @@ -19,3 +19,7 @@ def __init__( self.type = type self.objects = objects self.valid_only = valid_only + + def merge(self, other: "SaiCommon"): + super().merge(other) + self.__dict__.update(other.__dict__) \ No newline at end of file