diff --git a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java index bed70849..96b923eb 100644 --- a/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java +++ b/java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java @@ -52,6 +52,50 @@ public void onNewDataReceived(TensorsData data) { } }; + private Pipeline.NewDataCallback mCountCb = new Pipeline.NewDataCallback() { + @Override + public void onNewDataReceived(TensorsData data) { + mReceived++; + } + }; + + /** + * Runs a pipeline with the given custom-filter and returns the number of received data. + * The pipeline may fall into an error state while pushing the buffers, which is expected + * when the custom-filter returns invalid output data. + */ + private int runCustomFilterPipeline(String filterName, TensorsInfo info) throws Exception { + String desc = "appsrc name=srcx ! " + + "other/tensor,dimension=(string)10,type=(string)int32,framerate=(fraction)0/1 ! " + + "tensor_filter framework=custom-easy model=" + filterName + " ! " + + "tensor_sink name=sinkx"; + + try (Pipeline pipe = new Pipeline(desc)) { + pipe.registerSinkCallback("sinkx", mCountCb); + pipe.start(); + + for (int i = 0; i < 5; i++) { + try { + pipe.inputData("srcx", TensorsData.allocate(info)); + } catch (IllegalStateException e) { + /* expected, the pipeline is in an error state */ + } + + Thread.sleep(20); + } + + Thread.sleep(300); + + try { + pipe.stop(); + } catch (IllegalStateException e) { + /* expected, the pipeline is in an error state */ + } + } + + return mReceived; + } + private void registerCustomFilters() { try { TensorsInfo inputInfo = new TensorsInfo(); @@ -275,6 +319,78 @@ public TensorsData invoke(TensorsData in) { } } + @Test + public void testInvokeInvalidOutputSize_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + /* the size of the output data is larger than the registered output info */ + final TensorsInfo invalidInfo = new TensorsInfo(); + invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{20}); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-size", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + return TensorsData.allocate(invalidInfo); + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testInvokeInvalidOutputCount_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + /* the number of the output tensors is larger than the registered output info */ + final TensorsInfo invalidInfo = inputInfo.clone(); + invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-count", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + return TensorsData.allocate(invalidInfo); + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testInvokeClosedOutput_n() { + TensorsInfo inputInfo = new TensorsInfo(); + inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10}); + + TensorsInfo outputInfo = inputInfo.clone(); + + try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-closed", + inputInfo, outputInfo, new CustomFilter.Callback() { + @Override + public TensorsData invoke(TensorsData in) { + TensorsData out = TensorsData.allocate(in.getTensorsInfo()); + + /* the data list is empty while the tensors info is valid */ + out.close(); + return out; + } + })) { + assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo)); + } catch (Exception e) { + fail(); + } + } + @Test public void testRegisterNullName_n() { TensorsInfo inputInfo = new TensorsInfo(); diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java index c305409b..ae045171 100644 --- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java +++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java @@ -33,6 +33,11 @@ public interface Callback { * NNStreamer filter invokes the given custom-filter callback while processing the pipeline. * Note that, if it is unnecessary to execute the input data, return null to drop the buffer. * + * The returned data is copied into the output memory of the pipeline, so it should match + * the output information given to {@link #create(String, TensorsInfo, TensorsInfo, Callback)}. + * If the number of the tensors or the size of a buffer is different, the invoke fails and + * the pipeline stops with an error. + * * @param in The input data (a single frame, tensor/tensors) * * @return The output data (a single frame, tensor/tensors) diff --git a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c index bdefd0b5..85e7611f 100644 --- a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c +++ b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c @@ -489,13 +489,19 @@ nns_convert_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, /** * @brief Parse tensors data from TensorsData object. + * @param[in,out] data_h The tensors data handle. If it is NULL, a new handle is + * created from the given info and this function owns the + * cloned buffers. Otherwise the buffers belong to the + * caller and are only filled in (@a clone must be TRUE), + * so the data object must exactly match the number and + * the size of the tensors. */ gboolean nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, jobject obj_data, gboolean clone, const ml_tensors_info_h info_h, ml_tensors_data_h * data_h) { - guint i; + guint i, num_tensors; tensors_data_class_info_s *tensors_data_cls; ml_tensors_data_s *data; jobjectArray data_arr; @@ -507,6 +513,7 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, g_return_val_if_fail (env, FALSE); g_return_val_if_fail (obj_data, FALSE); g_return_val_if_fail (data_h, FALSE); + g_return_val_if_fail (*data_h == NULL || clone, FALSE); tensors_data_cls = &pipe_info->tensors_data_cls_info; @@ -542,7 +549,17 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, tensors_data_cls->mid_get_array); /* number of tensors data */ - data->num_tensors = (unsigned int) (*env)->GetArrayLength (env, data_arr); + num_tensors = (guint) (*env)->GetArrayLength (env, data_arr); + + if (num_tensors > ML_TENSOR_SIZE_LIMIT || + (!created && num_tensors != data->num_tensors)) { + _ml_loge ("The number of the tensors (%u) in the data object is invalid.", + num_tensors); + failed = TRUE; + goto done; + } + + data->num_tensors = num_tensors; /* set tensor data */ for (i = 0; i < data->num_tensors; i++) { @@ -553,14 +570,23 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env, gpointer data_ptr = (*env)->GetDirectBufferAddress (env, tensor); if (clone) { - if (data->tensors[i].data && data->tensors[i].size != data_size) { - _ml_logd ("The data size is not matched, reallocate new memory."); - g_clear_pointer (&data->tensors[i].data, g_free); + if (created) { + if (data->tensors[i].size != data_size) { + _ml_logd ("The data size is not matched, reallocate new memory."); + g_clear_pointer (&data->tensors[i].data, g_free); + } + + if (data->tensors[i].data == NULL) + data->tensors[i].data = g_malloc (data_size); + } else if (data->tensors[i].data == NULL || + data->tensors[i].size != data_size) { + _ml_loge ("The size of the tensor[%u] is %zd but %zd is expected.", + i, data_size, data->tensors[i].size); + (*env)->DeleteLocalRef (env, tensor); + failed = TRUE; + goto done; } - if (data->tensors[i].data == NULL) - data->tensors[i].data = g_malloc (data_size); - memcpy (data->tensors[i].data, data_ptr, data_size); } else { data->tensors[i].data = data_ptr; diff --git a/tests/capi/unittest_capi_inference.cc b/tests/capi/unittest_capi_inference.cc index d9afdee8..19eea8ed 100644 --- a/tests/capi/unittest_capi_inference.cc +++ b/tests/capi/unittest_capi_inference.cc @@ -7325,6 +7325,168 @@ TEST (nnstreamer_capi_custom, register_filter_01_p) g_free (filter_data_size); } +#define TEST_CUSTOM_OUT_SIZE (4U) +#define TEST_CUSTOM_OUT_VALUE (0x5A) + +/** + * @brief Data to check the output buffer of a custom-easy filter. + */ +typedef struct { + guint invoked; + guint received; + gboolean failed; +} custom_easy_out_data_s; + +/** + * @brief Invoke callback to check the output buffer given to a custom-easy filter. + */ +static int +test_custom_easy_out_cb (const ml_tensors_data_h in, ml_tensors_data_h out, void *user_data) +{ + custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data; + ml_tensors_info_h out_info = NULL; + void *raw_data = NULL; + size_t data_size = 0; + unsigned int count = 0; + guint i; + + G_LOCK (callback_lock); + result->invoked++; + + /* the number of the output tensors is fixed to the registered info. */ + if (ml_tensors_data_get_info (out, &out_info) != ML_ERROR_NONE + || ml_tensors_info_get_count (out_info, &count) != ML_ERROR_NONE || count != 1U) + result->failed = TRUE; + ml_tensors_info_destroy (out_info); + + /* the output buffer is allocated by the framework, fill it in-place. */ + if (ml_tensors_data_get_tensor_data (out, 0, &raw_data, &data_size) != ML_ERROR_NONE + || raw_data == NULL || data_size != TEST_CUSTOM_OUT_SIZE) { + result->failed = TRUE; + } else { + for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++) + ((guint8 *) raw_data)[i] = TEST_CUSTOM_OUT_VALUE; + } + G_UNLOCK (callback_lock); + + return 0; +} + +/** + * @brief Sink callback to check the output filled by a custom-easy filter. + */ +static void +test_custom_easy_out_sink_cb ( + const ml_tensors_data_h data, const ml_tensors_info_h info, void *user_data) +{ + custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data; + void *raw_data = NULL; + size_t data_size = 0; + guint i; + + G_LOCK (callback_lock); + result->received++; + + if (ml_tensors_data_get_tensor_data (data, 0, &raw_data, &data_size) != ML_ERROR_NONE + || data_size != TEST_CUSTOM_OUT_SIZE) { + result->failed = TRUE; + } else { + for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++) { + if (((guint8 *) raw_data)[i] != TEST_CUSTOM_OUT_VALUE) + result->failed = TRUE; + } + } + G_UNLOCK (callback_lock); +} + +/** + * @brief Test for the output buffer of a custom-easy filter. + * @detail The invoke callback gets the output memory owned by tensor_filter, + * sized by the registered output info. Bindings (e.g. Android JNI) copy + * the result into that memory, so releasing or replacing it is invalid. + */ +TEST (nnstreamer_capi_custom, invoke_output_buffer_p) +{ + const char test_custom_filter[] = "test-custom-filter-out"; + ml_pipeline_h pipe; + ml_pipeline_src_h src; + ml_pipeline_sink_h sink; + ml_custom_easy_filter_h custom; + ml_tensors_info_h in_info, out_info; + ml_tensors_data_h in_data; + ml_tensor_dimension in_dim = { 2, 1, 1, 1 }; + ml_tensor_dimension out_dim = { TEST_CUSTOM_OUT_SIZE, 1, 1, 1 }; + custom_easy_out_data_s *result; + int status; + guint i; + gchar *pipeline = g_strdup_printf ( + "appsrc name=srcx ! other/tensor,dimension=(string)2:1:1:1,type=(string)int8,framerate=(fraction)0/1 ! tensor_filter framework=custom-easy model=%s ! tensor_sink name=sinkx", + test_custom_filter); + + result = (custom_easy_out_data_s *) g_malloc0 (sizeof (custom_easy_out_data_s)); + + ml_tensors_info_create (&in_info); + ml_tensors_info_set_count (in_info, 1); + ml_tensors_info_set_tensor_type (in_info, 0, ML_TENSOR_TYPE_INT8); + ml_tensors_info_set_tensor_dimension (in_info, 0, in_dim); + + ml_tensors_info_create (&out_info); + ml_tensors_info_set_count (out_info, 1); + ml_tensors_info_set_tensor_type (out_info, 0, ML_TENSOR_TYPE_INT8); + ml_tensors_info_set_tensor_dimension (out_info, 0, out_dim); + + status = ml_pipeline_custom_easy_filter_register (test_custom_filter, in_info, + out_info, test_custom_easy_out_cb, result, &custom); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_construct (pipeline, NULL, NULL, &pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_sink_register ( + pipe, "sinkx", test_custom_easy_out_sink_cb, result, &sink); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_get_handle (pipe, "srcx", &src); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_start (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + for (i = 0; i < 5; i++) { + status = ml_tensors_data_create (in_info, &in_data); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_input_data (src, in_data, ML_PIPELINE_BUF_POLICY_AUTO_FREE); + EXPECT_EQ (status, ML_ERROR_NONE); + + g_usleep (50000); /* 50ms. Wait a bit. */ + } + + status = ml_pipeline_stop (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_src_release_handle (src); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_sink_unregister (sink); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_destroy (pipe); + EXPECT_EQ (status, ML_ERROR_NONE); + + status = ml_pipeline_custom_easy_filter_unregister (custom); + EXPECT_EQ (status, ML_ERROR_NONE); + + EXPECT_TRUE (result->invoked > 0U); + EXPECT_TRUE (result->received > 0U); + EXPECT_FALSE (result->failed); + + ml_tensors_info_destroy (in_info); + ml_tensors_info_destroy (out_info); + g_free (pipeline); + g_free (result); +} + /** * @brief Test for custom-easy registration. * @detail Invalid params.