Use opset15 version of Str Pack/Unpack

python/openvino_tokenizers/build_tokenizer.py

@@ -65,7 +65,7 @@ def build_rwkv_tokenizer(
     if clean_up_tokenization_spaces:
         RegexDecodingStep.clean_up_tokenization_spaces().get_ov_subgraph(detokenizer_output)
 
-    detokenizer_output = _get_factory().create("StringTensorPack", detokenizer_output).outputs()
+    detokenizer_output = _get_factory("opset15").create("StringTensorPack", detokenizer_output).outputs()
     detokenizer_output[0].tensor.add_names({STRING_OUTPUT_NAME})
 
     detokenizer = Model(detokenizer_output, [detokenizer_input], DETOKENIZER_NAME)

python/openvino_tokenizers/hf_parser.py

@@ -985,7 +985,7 @@ def get_sp_detokenizer(
     if params.utf8_replace_mode is not None and params.utf8_replace_mode != UTF8ReplaceMode.DISABLE:
         last_sinks = UTF8ValidateStep(params.utf8_replace_mode).get_ov_subgraph(detokenizer)
 
-    string_output = _get_factory().create("StringTensorPack", last_sinks).outputs()
+    string_output = _get_factory("opset15").create("StringTensorPack", last_sinks).outputs()
     string_output[0].tensor.add_names({STRING_OUTPUT_NAME})
     tokenizer_detokenizer = Model(string_output, [model_input], DETOKENIZER_NAME)
     tokenizer_detokenizer.validate_nodes_and_infer_types()

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -1274,7 +1274,7 @@ def create_decoding_pipeline(self, input_nodes: List[Output]) -> List[Output]:
             pipeline_step = step.get_ov_subgraph(input_nodes)
             input_nodes = pipeline_step
 
-        return _get_factory().create("StringTensorPack", input_nodes).outputs()
+        return _get_factory("opset15").create("StringTensorPack", input_nodes).outputs()
 
     def get_detokenizer_ov_subgraph(self) -> Model:
         self.finalize()

src/string_tensor_pack.hpp

@@ -5,20 +5,24 @@
 #pragma once
 
 #include <openvino/op/op.hpp>
+#include <openvino/op/string_tensor_pack.hpp>
 
 // Having a decomposed representation for a tensor, converts it to a single string tensor with element::string element type.
-class StringTensorPack : public ov::op::Op {
+class StringTensorPack : public ov::op::v15::StringTensorPack {
 public:
     OPENVINO_OP("StringTensorPack");
 
     StringTensorPack () = default;
 
     StringTensorPack(ov::OutputVector inputs, const std::string& mode = "begins_ends")
-        : ov::op::Op(inputs), m_mode(mode) {
+        : ov::op::v15::StringTensorPack(inputs[0], inputs[1], inputs[2]), m_mode(mode) {
         constructor_validate_and_infer_types();
     }
 
-    void validate_and_infer_types() override;
+    void validate_and_infer_types() override {
+        OPENVINO_ASSERT(m_mode == "begins_ends", "StringTensorUnpack supporst only 'begins_ends' mode, but get ", m_mode);
+        ov::op::v15::StringTensorPack::validate_and_infer_types();
+    }
 
     std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector& inputs) const override {
         auto result = std::make_shared<StringTensorPack>(inputs, m_mode);
@@ -30,12 +34,6 @@ class StringTensorPack : public ov::op::Op {
         return true;
     }
 
-    bool has_evaluate() const override {
-        return true;
-    }
-
-    bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override;
-
 private:
 
     std::string m_mode = "begins_ends";

src/string_tensor_unpack.hpp

@@ -5,23 +5,27 @@
 #pragma once
 
 #include <openvino/op/op.hpp>
+#include <openvino/op/string_tensor_unpack.hpp>
 
 // Unpack a string tensor representation regardless of the source format, which
 // can be an OpenVINO tensor with element::string element type or u8 legacy packed
 // representation, to a decompose tensor representation that may potentially
 // consist of multiple tensors. The destination format is defined by `mode` attribute.
-class StringTensorUnpack : public ov::op::Op {
+class StringTensorUnpack : public ov::op::v15::StringTensorUnpack {
 public:
     OPENVINO_OP("StringTensorUnpack");
 
     StringTensorUnpack () = default;
 
     StringTensorUnpack(ov::OutputVector inputs, const std::string& mode = "begins_ends")
-        : ov::op::Op(inputs), m_mode(mode) {
+        : ov::op::v15::StringTensorUnpack(inputs[0]), m_mode(mode) {
         constructor_validate_and_infer_types();
     }
 
-    void validate_and_infer_types() override;
+    void validate_and_infer_types() override {
+        OPENVINO_ASSERT(m_mode == "begins_ends", "StringTensorUnpack supporst only 'begins_ends' mode, but get ", m_mode);
+        ov::op::v15::StringTensorUnpack::validate_and_infer_types();
+    }
 
     std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector& inputs) const override {
         auto result = std::make_shared<StringTensorUnpack>(inputs, m_mode);
@@ -33,12 +37,6 @@ class StringTensorUnpack : public ov::op::Op {
         return true;
     }
 
-    bool has_evaluate() const override {
-        return true;
-    }
-
-    bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override;
-
 private:
 
     std::string m_mode = "begins_ends";

tests/layer_tests.py

@@ -71,9 +71,9 @@ def create_normalization_model(layer: Union[NormalizationStep, DecodingStep]) ->
     input_node = op.Parameter(Type.string, PartialShape(["?"]))
     input_node.set_friendly_name("string_input")
 
-    output = _get_factory().create("StringTensorUnpack", input_node.outputs()).outputs()
+    output = _get_factory("opset15").create("StringTensorUnpack", input_node.outputs()).outputs()
     output = layer.get_ov_subgraph(output)
-    output = _get_factory().create("StringTensorPack", output).outputs()
+    output = _get_factory("opset15").create("StringTensorPack", output).outputs()
     normalizer = Model(output, [input_node], "normalizer")
 
     return core.compile_model(normalizer)
@@ -179,10 +179,10 @@ def create_splitting_model(layer: PreTokenizatinStep) -> ov.CompiledModel:
     input_node = op.Parameter(Type.string, PartialShape(["?"]))
     input_node.set_friendly_name("string_input")
 
-    output = _get_factory().create("StringTensorUnpack", input_node.outputs()).outputs()
+    output = _get_factory("opset15").create("StringTensorUnpack", input_node.outputs()).outputs()
     output = TokenizerPipeline.add_ragged_dimension(output)
     output = layer.get_ov_subgraph(output)
-    output = _get_factory().create("StringTensorPack", output[2:5]).outputs()
+    output = _get_factory("opset15").create("StringTensorPack", output[2:5]).outputs()
     splitter = Model(output, [input_node], "splitter")
 
     return core.compile_model(splitter)