From 394979f45d711db5811605bedb74c1edcb0e2e8b Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Thu, 10 Sep 2026 15:14:17 +0900 Subject: [PATCH 1/3] [Android] do not release the tensors data owned by the caller nns_parse_tensors_data() may be given a data handle that already holds buffers. The only such caller is nns_customfilter_invoke(), where the handle wraps the output memory of tensor_filter: it is mapped GstMemory, not a GLib allocation, and the framework unmaps and keeps using it after the callback returns. The clone path treated those buffers as its own. When the TensorsData returned by CustomFilter.invoke() was built from a tensors info other than the registered output info, the sizes did not match, so the mapped memory was passed to g_free() and a replacement was allocated. That is an invalid free (heap corruption), the framework never sees the replacement, and GStreamer later unmaps memory this code already released. A Java list longer than the registered output info allocated and leaked a buffer on every invoke, one longer than ML_TENSOR_SIZE_LIMIT wrote past the fixed tensors array, and an empty list (a closed TensorsData) passed the parse with nothing copied, so an unfilled frame went downstream. Split the two ownership cases. A handle created here still allocates and clones as before. A handle given by the caller is only filled in: the number of the tensors and every buffer size must match, and a mismatch now fails the parse so the custom-filter invoke returns an error instead of corrupting the heap. The tensor count is bounded in both cases. A caller-owned handle is accepted only together with clone, because the direct-access path would overwrite the caller's pointers. No caller does that today; the check keeps it that way. Fixes the item H4 of nnstreamer/api#690. Signed-off-by: MyungJoo Ham Co-Authored-By: Claude Opus 5 --- .../src/main/jni/nnstreamer-native-api.c | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c index bdefd0b5..85e7611f 100644 --- a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c +++ b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c @@ -489,13 +489,19 @@ nns_convert_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, /** * @brief Parse tensors data from TensorsData object. + * @param[in,out] data_h The tensors data handle. If it is NULL, a new handle is + * created from the given info and this function owns the + * cloned buffers. Otherwise the buffers belong to the + * caller and are only filled in (@a clone must be TRUE), + * so the data object must exactly match the number and + * the size of the tensors. */ gboolean nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, jobject obj_data, gboolean clone, const ml_tensors_info_h info_h, ml_tensors_data_h * data_h) { - guint i; + guint i, num_tensors; tensors_data_class_info_s *tensors_data_cls; ml_tensors_data_s *data; jobjectArray data_arr; @@ -507,6 +513,7 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, g_return_val_if_fail (env, FALSE); g_return_val_if_fail (obj_data, FALSE); g_return_val_if_fail (data_h, FALSE); + g_return_val_if_fail (*data_h == NULL || clone, FALSE); tensors_data_cls = &pipe_info->tensors_data_cls_info; @@ -542,7 +549,17 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, tensors_data_cls->mid_get_array); /* number of tensors data */ - data->num_tensors = (unsigned int) (*env)->GetArrayLength (env, data_arr); + num_tensors = (guint) (*env)->GetArrayLength (env, data_arr); + + if (num_tensors > ML_TENSOR_SIZE_LIMIT || + (!created && num_tensors != data->num_tensors)) { + _ml_loge ("The number of the tensors (%u) in the data object is invalid.", + num_tensors); + failed = TRUE; + goto done; + } + + data->num_tensors = num_tensors; /* set tensor data */ for (i = 0; i < data->num_tensors; i++) { @@ -553,14 +570,23 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, gpointer data_ptr = (*env)->GetDirectBufferAddress (env, tensor); if (clone) { - if (data->tensors[i].data && data->tensors[i].size != data_size) { - _ml_logd ("The data size is not matched, reallocate new memory."); - g_clear_pointer (&data->tensors[i].data, g_free); + if (created) { + if (data->tensors[i].size != data_size) { + _ml_logd ("The data size is not matched, reallocate new memory."); + g_clear_pointer (&data->tensors[i].data, g_free); + } + + if (data->tensors[i].data == NULL) + data->tensors[i].data = g_malloc (data_size); + } else if (data->tensors[i].data == NULL || + data->tensors[i].size != data_size) { + _ml_loge ("The size of the tensor[%u] is %zd but %zd is expected.", + i, data_size, data->tensors[i].size); + (*env)->DeleteLocalRef (env, tensor); + failed = TRUE; + goto done; } - if (data->tensors[i].data == NULL) - data->tensors[i].data = g_malloc (data_size); - memcpy (data->tensors[i].data, data_ptr, data_size); } else { data->tensors[i].data = data_ptr; From baadfb54cdf39bd297c1457491aa90b6999a0030 Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Thu, 10 Sep 2026 15:14:26 +0900 Subject: [PATCH 2/3] [Test] cover the output buffer contract of a custom-filter The Android negative cases drive a CustomFilter whose invoke() returns a TensorsData that does not match the registered output info. Before the previous commit each of them mishandled the output memory of the framework: the larger buffer passed the mapped memory to g_free(), the extra tensor allocated and leaked a buffer on every invoke, and the closed (empty) object passed the parse with nothing copied, so an unfilled frame reached the sink. Now the invoke fails for all three and nothing reaches the sink. The C unittest locks down what the JNI binding relies on: the invoke callback of a custom-easy filter is handed an output handle whose tensor count and buffer sizes come from the registered output info, with the memory already allocated by tensor_filter, and the data written into that memory is what the sink receives. It runs in CI, so a change that hands the callback a differently owned output buffer fails there rather than only on a device. Signed-off-by: MyungJoo Ham Co-Authored-By: Claude Opus 5 --- .../nnstreamer/APITestCustomFilter.java | 119 +++++++++++++ tests/capi/unittest_capi_inference.cc | 163 ++++++++++++++++++ 2 files changed, 282 insertions(+) diff --git a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java index bed70849..56a8c687 100644 --- a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java +++ b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java @@ -52,6 +52,50 @@ public void onNewDataReceived(TensorsData data) { } }; + private Pipeline.NewDataCallback mCountCb = new Pipeline.NewDataCallback() { + @Override + public void onNewDataReceived(TensorsData data) { + mReceived++; + } + }; + + /** + * Runs a pipeline with the given custom-filter and returns the number of received data. + * The pipeline may fall into an error state while pushing the buffers, which is expected + * when the custom-filter returns invalid output data. + */ + private int runCustomFilterPipeline(String filterName, TensorsInfo info) throws Exception { + String desc = "appsrc name=srcx ! " + + "other/tensor,dimension=(string)10,type=(string)int32,framerate=(fraction)0/1 ! " + + "tensor_filter framework=custom-easy model=" + filterName + " ! " + + "tensor_sink name=sinkx"; + + try (Pipeline pipe = new Pipeline(desc)) { + pipe.registerSinkCallback("sinkx", mCountCb); + pipe.start(); + + for (int i = 0; i < 5; i++) { + try { + pipe.inputData("srcx", TensorsData.allocate(info)); + } catch (IllegalStateException e) { + /* expected, the pipeline is in an error state */ + } + + Thread.sleep(20); + } + + Thread.sleep(300); + + try { + pipe.stop(); + } catch (IllegalStateException e) { + /* expected, the pipeline is in an error state */ + } + } + + return mReceived; + } + private void registerCustomFilters() { try { TensorsInfo inputInfo = new TensorsInfo(); @@ -275,6 +319,81 @@ public TensorsData invoke(TensorsData in) { } } + @Test + public void testInvokeInvalidOutputSize_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + /* the size of the output data is larger than the registered output info */ + final TensorsInfo invalidInfo = new TensorsInfo(); + invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{20}); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-size", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + return TensorsData.allocate(invalidInfo); + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + assertFalse(mInvalidState); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testInvokeInvalidOutputCount_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + /* the number of the output tensors is larger than the registered output info */ + final TensorsInfo invalidInfo = inputInfo.clone(); + invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-count", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + return TensorsData.allocate(invalidInfo); + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + assertFalse(mInvalidState); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testInvokeClosedOutput_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-closed", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + TensorsData out = TensorsData.allocate(in.getTensorsInfo()); + + /* the data list is empty while the tensors info is valid */ + out.close(); + return out; + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + assertFalse(mInvalidState); + } catch (Exception e) { + fail(); + } + } + @Test public void testRegisterNullName_n() { TensorsInfo inputInfo = new TensorsInfo(); diff --git a/tests/capi/unittest_capi_inference.cc b/tests/capi/unittest_capi_inference.cc index d9afdee8..e8937935 100644 --- a/tests/capi/unittest_capi_inference.cc +++ b/tests/capi/unittest_capi_inference.cc @@ -7325,6 +7325,169 @@ TEST (nnstreamer_capi_custom, register_filter_01_p) g_free (filter_data_size); } +#define TEST_CUSTOM_OUT_SIZE (4U) +#define TEST_CUSTOM_OUT_VALUE (0x5A) + +/** + * @brief Data to check the output buffer of a custom-easy filter. + */ +typedef struct { + guint invoked; + guint received; + gboolean failed; +} custom_easy_out_data_s; + +/** + * @brief Invoke callback to check the output buffer given to a custom-easy filter. + */ +static int +test_custom_easy_out_cb (const ml_tensors_data_h in, ml_tensors_data_h out, void *user_data) +{ + custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data; + ml_tensors_info_h out_info = NULL; + void *raw_data = NULL; + size_t data_size = 0; + unsigned int count = 0; + guint i; + + G_LOCK (callback_lock); + result->invoked++; + + /* the number of the output tensors is fixed to the registered info. */ + if (ml_tensors_data_get_info (out, &out_info) != ML_ERROR_NONE + || ml_tensors_info_get_count (out_info, &count) != ML_ERROR_NONE + || count != 1U) + result->failed = TRUE; + ml_tensors_info_destroy (out_info); + + /* the output buffer is allocated by the framework, fill it in-place. */ + if (ml_tensors_data_get_tensor_data (out, 0, &raw_data, &data_size) != ML_ERROR_NONE + || raw_data == NULL || data_size != TEST_CUSTOM_OUT_SIZE) { + result->failed = TRUE; + } else { + for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++) + ((guint8 *) raw_data)[i] = TEST_CUSTOM_OUT_VALUE; + } + G_UNLOCK (callback_lock); + + return 0; +} + +/** + * @brief Sink callback to check the output filled by a custom-easy filter. + */ +static void +test_custom_easy_out_sink_cb (const ml_tensors_data_h data, + const ml_tensors_info_h info, void *user_data) +{ + custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data; + void *raw_data = NULL; + size_t data_size = 0; + guint i; + + G_LOCK (callback_lock); + result->received++; + + if (ml_tensors_data_get_tensor_data (data, 0, &raw_data, &data_size) != ML_ERROR_NONE + || data_size != TEST_CUSTOM_OUT_SIZE) { + result->failed = TRUE; + } else { + for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++) { + if (((guint8 *) raw_data)[i] != TEST_CUSTOM_OUT_VALUE) + result->failed = TRUE; + } + } + G_UNLOCK (callback_lock); +} + +/** + * @brief Test for the output buffer of a custom-easy filter. + * @detail The invoke callback gets the output memory owned by tensor_filter, + * sized by the registered output info. Bindings (e.g. Android JNI) copy + * the result into that memory, so releasing or replacing it is invalid. + */ +TEST (nnstreamer_capi_custom, invoke_output_buffer_p) +{ + const char test_custom_filter[] = "test-custom-filter-out"; + ml_pipeline_h pipe; + ml_pipeline_src_h src; + ml_pipeline_sink_h sink; + ml_custom_easy_filter_h custom; + ml_tensors_info_h in_info, out_info; + ml_tensors_data_h in_data; + ml_tensor_dimension in_dim = { 2, 1, 1, 1 }; + ml_tensor_dimension out_dim = { TEST_CUSTOM_OUT_SIZE, 1, 1, 1 }; + custom_easy_out_data_s *result; + int status; + guint i; + gchar *pipeline = g_strdup_printf ( + "appsrc name=srcx ! other/tensor,dimension=(string)2:1:1:1,type=(string)int8,framerate=(fraction)0/1 ! tensor_filter framework=custom-easy model=%s ! tensor_sink name=sinkx", + test_custom_filter); + + result = (custom_easy_out_data_s *) g_malloc0 (sizeof (custom_easy_out_data_s)); + + ml_tensors_info_create (&in_info); + ml_tensors_info_set_count (in_info, 1); + ml_tensors_info_set_tensor_type (in_info, 0, ML_TENSOR_TYPE_INT8); + ml_tensors_info_set_tensor_dimension (in_info, 0, in_dim); + + ml_tensors_info_create (&out_info); + ml_tensors_info_set_count (out_info, 1); + ml_tensors_info_set_tensor_type (out_info, 0, ML_TENSOR_TYPE_INT8); + ml_tensors_info_set_tensor_dimension (out_info, 0, out_dim); + + status = ml_pipeline_custom_easy_filter_register (test_custom_filter, in_info, + out_info, test_custom_easy_out_cb, result, &custom); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_construct (pipeline, NULL, NULL, &pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_sink_register ( + pipe, "sinkx", test_custom_easy_out_sink_cb, result, &sink); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_get_handle (pipe, "srcx", &src); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_start (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + for (i = 0; i < 5; i++) { + status = ml_tensors_data_create (in_info, &in_data); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_input_data (src, in_data, ML_PIPELINE_BUF_POLICY_AUTO_FREE); + EXPECT_EQ (status, ML_ERROR_NONE); + + g_usleep (50000); /* 50ms. Wait a bit. */ + } + + status = ml_pipeline_stop (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_release_handle (src); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_sink_unregister (sink); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_destroy (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_custom_easy_filter_unregister (custom); + EXPECT_EQ (status, ML_ERROR_NONE); + + EXPECT_TRUE (result->invoked > 0U); + EXPECT_TRUE (result->received > 0U); + EXPECT_FALSE (result->failed); + + ml_tensors_info_destroy (in_info); + ml_tensors_info_destroy (out_info); + g_free (pipeline); + g_free (result); +} + /** * @brief Test for custom-easy registration. * @detail Invalid params. From 7d792b7f33eb84dd00881c9b2e64c380533dfc9b Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Thu, 10 Sep 2026 15:14:37 +0900 Subject: [PATCH 3/3] [Android] document the output contract of a custom-filter callback Say in CustomFilter.Callback.invoke() that the returned data is copied into the output memory of the pipeline, so it has to match the output information passed to create(), and that a mismatch fails the invoke and stops the pipeline with an error. That is not the per-buffer drop of returning null: tensor_filter turns a failed invoke into GST_FLOW_ERROR. The requirement is now enforced by the JNI layer, so it belongs in the API documentation. Also drop the assertions on mInvalidState from the new negative cases: they run with a sink callback that only counts, so the flag can never be set and the check cannot fail. The number of the received data is the assertion that matters. The remaining hunk reflows the new unittest to what clang-format wants. Signed-off-by: MyungJoo Ham Co-Authored-By: Claude Opus 5 --- .../java/org/nnsuite/nnstreamer/APITestCustomFilter.java | 3 --- .../src/main/java/org/nnsuite/nnstreamer/CustomFilter.java | 5 +++++ tests/capi/unittest_capi_inference.cc | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java index 56a8c687..96b923eb 100644 --- a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java +++ b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java @@ -338,7 +338,6 @@ public TensorsData invoke(TensorsData in) { } })) { assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); - assertFalse(mInvalidState); } catch (Exception e) { fail(); } @@ -363,7 +362,6 @@ public TensorsData invoke(TensorsData in) { } })) { assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); - assertFalse(mInvalidState); } catch (Exception e) { fail(); } @@ -388,7 +386,6 @@ public TensorsData invoke(TensorsData in) { } })) { assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); - assertFalse(mInvalidState); } catch (Exception e) { fail(); } diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java index c305409b..ae045171 100644 --- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java +++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java @@ -33,6 +33,11 @@ public interface Callback { * NNStreamer filter invokes the given custom-filter callback while processing the pipeline. * Note that, if it is unnecessary to execute the input data, return null to drop the buffer. * + * The returned data is copied into the output memory of the pipeline, so it should match + * the output information given to {@link #create(String, TensorsInfo, TensorsInfo, Callback)}. + * If the number of the tensors or the size of a buffer is different, the invoke fails and + * the pipeline stops with an error. + * * @param in The input data (a single frame, tensor/tensors) * * @return The output data (a single frame, tensor/tensors) diff --git a/tests/capi/unittest_capi_inference.cc b/tests/capi/unittest_capi_inference.cc index e8937935..19eea8ed 100644 --- a/tests/capi/unittest_capi_inference.cc +++ b/tests/capi/unittest_capi_inference.cc @@ -7355,8 +7355,7 @@ test_custom_easy_out_cb (const ml_tensors_data_h in, ml_tensors_data_h out, void /* the number of the output tensors is fixed to the registered info. */ if (ml_tensors_data_get_info (out, &out_info) != ML_ERROR_NONE - || ml_tensors_info_get_count (out_info, &count) != ML_ERROR_NONE - || count != 1U) + || ml_tensors_info_get_count (out_info, &count) != ML_ERROR_NONE || count != 1U) result->failed = TRUE; ml_tensors_info_destroy (out_info); @@ -7377,8 +7376,8 @@ test_custom_easy_out_cb (const ml_tensors_data_h in, ml_tensors_data_h out, void * @brief Sink callback to check the output filled by a custom-easy filter. */ static void -test_custom_easy_out_sink_cb (const ml_tensors_data_h data, - const ml_tensors_info_h info, void *user_data) +test_custom_easy_out_sink_cb ( + const ml_tensors_data_h data, const ml_tensors_info_h info, void *user_data) { custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data; void *raw_data = NULL;