diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp
index 3dd3425c99e8..16f059380759 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.cpp
+++ b/ggml/src/ggml-openvino/ggml-decoder.cpp
@@ -25,7 +25,6 @@
 #include <openvino/core/type/float16.hpp>
 #include <openvino/op/constant.hpp>
 #include <openvino/op/convert.hpp>
-#include <openvino/op/parameter.hpp>
 #include <openvino/runtime/tensor.hpp>
 #include <ostream>
 #include <set>
@@ -708,21 +707,7 @@ void GgmlOvDecoder::add_extra_inputs() {
     // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
 
     auto create_1d_input = [this](const std::string & name, int64_t value) {
-        if (m_is_static) {
-            auto constant =
-                std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{1}, std::vector<int64_t>{value});
-            constant->set_friendly_name(name);
-            m_model_extra_inputs[name] = constant;
-        } else {
-            auto param_node = std::make_shared<ov::op::v0::Parameter>(ov::element::i64, ov::Shape{1});
-            param_node->set_friendly_name(name);
-            param_node->output(0).get_tensor().set_names({name});
-            m_model_extra_inputs[name] = param_node;
-
-            auto tensor = std::make_shared<ov::Tensor>(ov::element::i64, ov::Shape{1});
-            *tensor->data<int64_t>() = value;
-            m_model_extra_input_values[name] = tensor;
-        }
+        m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, !m_is_static};
     };
 
     if (m_compute_params.attention_size != -1) {
@@ -773,11 +758,8 @@ void GgmlOvDecoder::compute_model_inputs() {
             std::string node_name = get_tensor_ov_name(m_cgraph, node);
             if (m_model_weights.find(node_name) == m_model_weights.end()) {
                 m_inputs[node_name] = node;
-                auto param_node = std::make_shared<ov::op::v0::Parameter>(
-                    get_ov_type(node), get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node]));
-                param_node->set_friendly_name(node_name);
-                param_node->output(0).get_tensor().set_names({node_name});
-                m_model_inputs[node_name] = param_node;
+                m_model_inputs[node_name] = {get_ov_type(node),
+                                             get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])};
             }
             continue;
         }
@@ -824,11 +806,8 @@ void GgmlOvDecoder::compute_model_inputs() {
                 src_name = get_tensor_ov_name(m_cgraph, src);
             }
             m_inputs[src_name] = src;
-            ov::PartialShape param_shape = get_graph_input_shape(node, src, m_node_dynamic_dims[src]);
-            auto param_node = std::make_shared<ov::op::v0::Parameter>(get_ov_type(src), param_shape);
-            param_node->set_friendly_name(src_name);
-            param_node->output(0).get_tensor().set_names({src_name});
-            m_model_inputs[src_name] = param_node;
+            m_model_inputs[src_name] = {get_ov_type(src),
+                                        get_graph_input_shape(node, src, m_node_dynamic_dims[src])};
         }
     }
 }
diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h
index 4911c069c033..1372d3ebdfb0 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.h
+++ b/ggml/src/ggml-openvino/ggml-decoder.h
@@ -207,18 +207,14 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelInputInfo> & get_model_inputs() const override {
         return m_model_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_extra_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> & get_model_extra_inputs() const override {
         return m_model_extra_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Tensor>> & get_model_extra_input_values() const {
-        return m_model_extra_input_values;
-    }
-
     virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const override {
         return m_model_weights;
     }
@@ -379,9 +375,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     ggml_cgraph * m_cgraph = nullptr;
     std::map<std::string, ggml_tensor *> m_inputs;
 
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_inputs;
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_extra_inputs;
-    std::map<std::string, std::shared_ptr<ov::Tensor>> m_model_extra_input_values;
+    std::map<std::string, ov::frontend::ggml::ModelInputInfo> m_model_inputs;
+    std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> m_model_extra_inputs;
     std::map<std::string, std::shared_ptr<ov::Node>> m_model_weights;
     std::map<std::string, ggml_tensor *> m_model_outputs;
     std::set<std::string> m_model_output_names;
diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h
index 7ec6bdd8b40e..ec6975282a54 100644
--- a/ggml/src/ggml-openvino/openvino/decoder.h
+++ b/ggml/src/ggml-openvino/openvino/decoder.h
@@ -6,12 +6,25 @@
 #include <openvino/core/partial_shape.hpp>
 #include <openvino/core/shape.hpp>
 #include <openvino/frontend/decoder.hpp>
+#include <set>
 #include <string>
 
 namespace ov {
 namespace frontend {
 namespace ggml {
 
+struct ModelInputInfo {
+    element::Type type;
+    PartialShape shape;
+};
+
+struct ModelExtraInputInfo {
+    element::Type type;
+    Shape shape;
+    int64_t value;
+    bool is_parameter;
+};
+
 class GgmlDecoder : public DecoderBase {
 public:
     virtual ov::Any get_attribute(const std::string & name) const = 0;
@@ -91,9 +104,9 @@ class GgmlDecoder : public DecoderBase {
 
     virtual int get_op_case(int node_idx) const = 0;
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_inputs() const = 0;
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_extra_inputs() const = 0;
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>>& get_model_weights() const = 0;
+    virtual const std::map<std::string, ModelInputInfo> & get_model_inputs() const = 0;
+    virtual const std::map<std::string, ModelExtraInputInfo> & get_model_extra_inputs() const = 0;
+    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const = 0;
     virtual std::set<std::string> get_model_output_names() const = 0;
 
     virtual int32_t * get_rope_params() const = 0;
diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp
index 1bf9135742e1..b0cc142a86d7 100644
--- a/ggml/src/ggml-openvino/openvino/translate_session.cpp
+++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp
@@ -44,6 +44,28 @@ using namespace ov::op;
 
 namespace {
 
+std::shared_ptr<ov::op::v0::Parameter> create_parameter(const std::string & name,
+                                                        const ModelInputInfo & input_info) {
+    auto param_node = std::make_shared<ov::op::v0::Parameter>(input_info.type, input_info.shape);
+    param_node->set_friendly_name(name);
+    param_node->output(0).get_tensor().set_names({name});
+    return param_node;
+}
+
+std::shared_ptr<ov::Node> create_extra_input(const std::string & name, const ModelExtraInputInfo & input_info) {
+    if (input_info.is_parameter) {
+        auto param_node = std::make_shared<ov::op::v0::Parameter>(input_info.type, input_info.shape);
+        param_node->set_friendly_name(name);
+        param_node->output(0).get_tensor().set_names({name});
+        return param_node;
+    }
+
+    auto constant = std::make_shared<ov::op::v0::Constant>(input_info.type, input_info.shape,
+                                                          std::vector<int64_t>{input_info.value});
+    constant->set_friendly_name(name);
+    return constant;
+}
+
 ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs(
     const std::shared_ptr<ov::Model> & model,
     const std::map<std::string, std::string> & kv_param_res_names) {
@@ -177,15 +199,17 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
     std::shared_ptr<GgmlDecoder> ggml_model_decoder = ggml_model->get_model_decoder();
 
     for (const auto & it : ggml_model_decoder->get_model_inputs()) {
-        params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second));
-        (*tensor_map)[it.first] = it.second;
+        auto param_node = create_parameter(it.first, it.second);
+        params.push_back(param_node);
+        (*tensor_map)[it.first] = param_node;
     }
 
     for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) {
-        if (std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second)) {
-            params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(it.second));
+        auto input_node = create_extra_input(it.first, it.second);
+        if (it.second.is_parameter) {
+            params.push_back(std::dynamic_pointer_cast<ov::op::v0::Parameter>(input_node));
         }
-        (*tensor_map)[it.first] = it.second;
+        (*tensor_map)[it.first] = input_node;
     }
 
     for (const auto & it : ggml_model_decoder->get_model_weights()) {
diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp
index b8e7af3d69c5..28a8b68c80ab 100644
--- a/ggml/src/ggml-openvino/utils.cpp
+++ b/ggml/src/ggml-openvino/utils.cpp
@@ -847,8 +847,10 @@ ov::Tensor convert_ggml_input_to_ov(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
 
 ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name) {
     ov::Tensor input_tensor;
-    if (ggml_decoder->get_model_extra_inputs().find(param_name) != ggml_decoder->get_model_extra_inputs().end()) {
-        input_tensor = *ggml_decoder->get_model_extra_input_values().at(param_name);
+    auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name);
+    if (extra_input != ggml_decoder->get_model_extra_inputs().end()) {
+        input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape);
+        *input_tensor.data<int64_t>() = extra_input->second.value;
     } else {
         input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name);
     }