[Android] Do not release the custom-filter output memory owned by tensor_filter

java/android/nnstreamer/src/androidTest/java/org/nnsuite/nnstreamer/APITestCustomFilter.java

@@ -52,6 +52,50 @@ public void onNewDataReceived(TensorsData data) {
         }
     };
 
+    private Pipeline.NewDataCallback mCountCb = new Pipeline.NewDataCallback() {
+        @Override
+        public void onNewDataReceived(TensorsData data) {
+            mReceived++;
+        }
+    };
+
+    /**
+     * Runs a pipeline with the given custom-filter and returns the number of received data.
+     * The pipeline may fall into an error state while pushing the buffers, which is expected
+     * when the custom-filter returns invalid output data.
+     */
+    private int runCustomFilterPipeline(String filterName, TensorsInfo info) throws Exception {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)10,type=(string)int32,framerate=(fraction)0/1 ! " +
+                "tensor_filter framework=custom-easy model=" + filterName + " ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            pipe.registerSinkCallback("sinkx", mCountCb);
+            pipe.start();
+
+            for (int i = 0; i < 5; i++) {
+                try {
+                    pipe.inputData("srcx", TensorsData.allocate(info));
+                } catch (IllegalStateException e) {
+                    /* expected, the pipeline is in an error state */
+                }
+
+                Thread.sleep(20);
+            }
+
+            Thread.sleep(300);
+
+            try {
+                pipe.stop();
+            } catch (IllegalStateException e) {
+                /* expected, the pipeline is in an error state */
+            }
+        }
+
+        return mReceived;
+    }
+
     private void registerCustomFilters() {
         try {
             TensorsInfo inputInfo = new TensorsInfo();
@@ -275,6 +319,78 @@ public TensorsData invoke(TensorsData in) {
         }
     }
 
+    @Test
+    public void testInvokeInvalidOutputSize_n() {
+        TensorsInfo inputInfo = new TensorsInfo();
+        inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10});
+
+        TensorsInfo outputInfo = inputInfo.clone();
+
+        /* the size of the output data is larger than the registered output info */
+        final TensorsInfo invalidInfo = new TensorsInfo();
+        invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{20});
+
+        try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-size",
+                inputInfo, outputInfo, new CustomFilter.Callback() {
+            @Override
+            public TensorsData invoke(TensorsData in) {
+                return TensorsData.allocate(invalidInfo);
+            }
+        })) {
+            assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo));
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void testInvokeInvalidOutputCount_n() {
+        TensorsInfo inputInfo = new TensorsInfo();
+        inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10});
+
+        TensorsInfo outputInfo = inputInfo.clone();
+
+        /* the number of the output tensors is larger than the registered output info */
+        final TensorsInfo invalidInfo = inputInfo.clone();
+        invalidInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10});
+
+        try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-count",
+                inputInfo, outputInfo, new CustomFilter.Callback() {
+            @Override
+            public TensorsData invoke(TensorsData in) {
+                return TensorsData.allocate(invalidInfo);
+            }
+        })) {
+            assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo));
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void testInvokeClosedOutput_n() {
+        TensorsInfo inputInfo = new TensorsInfo();
+        inputInfo.addTensorInfo(NNStreamer.TensorType.INT32, new int[]{10});
+
+        TensorsInfo outputInfo = inputInfo.clone();
+
+        try (CustomFilter customInvalid = CustomFilter.create("custom-invalid-closed",
+                inputInfo, outputInfo, new CustomFilter.Callback() {
+            @Override
+            public TensorsData invoke(TensorsData in) {
+                TensorsData out = TensorsData.allocate(in.getTensorsInfo());
+
+                /* the data list is empty while the tensors info is valid */
+                out.close();
+                return out;
+            }
+        })) {
+            assertEquals(0, runCustomFilterPipeline(customInvalid.getName(), inputInfo));
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
     @Test
     public void testRegisterNullName_n() {
         TensorsInfo inputInfo = new TensorsInfo();

java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/CustomFilter.java

@@ -33,6 +33,11 @@ public interface Callback {
          * NNStreamer filter invokes the given custom-filter callback while processing the pipeline.
          * Note that, if it is unnecessary to execute the input data, return null to drop the buffer.
          *
+         * The returned data is copied into the output memory of the pipeline, so it should match
+         * the output information given to {@link #create(String, TensorsInfo, TensorsInfo, Callback)}.
+         * If the number of the tensors or the size of a buffer is different, the invoke fails and
+         * the pipeline stops with an error.
+         *
          * @param in The input data (a single frame, tensor/tensors)
          *
          * @return The output data (a single frame, tensor/tensors)

java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c

@@ -489,13 +489,19 @@ nns_convert_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env,
 
 /**
  * @brief Parse tensors data from TensorsData object.
+ * @param[in,out] data_h The tensors data handle. If it is NULL, a new handle is
+ *                       created from the given info and this function owns the
+ *                       cloned buffers. Otherwise the buffers belong to the
+ *                       caller and are only filled in (@a clone must be TRUE),
+ *                       so the data object must exactly match the number and
+ *                       the size of the tensors.
  */
 gboolean
 nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env,
     jobject obj_data, gboolean clone, const ml_tensors_info_h info_h,
     ml_tensors_data_h * data_h)
 {
-  guint i;
+  guint i, num_tensors;
   tensors_data_class_info_s *tensors_data_cls;
   ml_tensors_data_s *data;
   jobjectArray data_arr;
@@ -507,6 +513,7 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env,
   g_return_val_if_fail (env, FALSE);
   g_return_val_if_fail (obj_data, FALSE);
   g_return_val_if_fail (data_h, FALSE);
+  g_return_val_if_fail (*data_h == NULL || clone, FALSE);
 
   tensors_data_cls = &pipe_info->tensors_data_cls_info;
 
@@ -542,7 +549,17 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env,
       tensors_data_cls->mid_get_array);
 
   /* number of tensors data */
-  data->num_tensors = (unsigned int) (*env)->GetArrayLength (env, data_arr);
+  num_tensors = (guint) (*env)->GetArrayLength (env, data_arr);
+
+  if (num_tensors > ML_TENSOR_SIZE_LIMIT ||
+      (!created && num_tensors != data->num_tensors)) {
+    _ml_loge ("The number of the tensors (%u) in the data object is invalid.",
+        num_tensors);
+    failed = TRUE;
+    goto done;
+  }
+
+  data->num_tensors = num_tensors;
 
   /* set tensor data */
   for (i = 0; i < data->num_tensors; i++) {
@@ -553,14 +570,23 @@ nns_parse_tensors_data (pipeline_info_s * pipe_info, JNIEnv * env,
       gpointer data_ptr = (*env)->GetDirectBufferAddress (env, tensor);
 
       if (clone) {
-        if (data->tensors[i].data && data->tensors[i].size != data_size) {
-          _ml_logd ("The data size is not matched, reallocate new memory.");
-          g_clear_pointer (&data->tensors[i].data, g_free);
+        if (created) {
+          if (data->tensors[i].size != data_size) {
+            _ml_logd ("The data size is not matched, reallocate new memory.");
+            g_clear_pointer (&data->tensors[i].data, g_free);
+          }
+
+          if (data->tensors[i].data == NULL)
+            data->tensors[i].data = g_malloc (data_size);
+        } else if (data->tensors[i].data == NULL ||
+            data->tensors[i].size != data_size) {
+          _ml_loge ("The size of the tensor[%u] is %zd but %zd is expected.",
+              i, data_size, data->tensors[i].size);
+          (*env)->DeleteLocalRef (env, tensor);
+          failed = TRUE;
+          goto done;
         }
 
-        if (data->tensors[i].data == NULL)
-          data->tensors[i].data = g_malloc (data_size);
-
         memcpy (data->tensors[i].data, data_ptr, data_size);
       } else {
         data->tensors[i].data = data_ptr;

tests/capi/unittest_capi_inference.cc

@@ -7325,6 +7325,168 @@ TEST (nnstreamer_capi_custom, register_filter_01_p)
   g_free (filter_data_size);
 }
 
+#define TEST_CUSTOM_OUT_SIZE (4U)
+#define TEST_CUSTOM_OUT_VALUE (0x5A)
+
+/**
+ * @brief Data to check the output buffer of a custom-easy filter.
+ */
+typedef struct {
+  guint invoked;
+  guint received;
+  gboolean failed;
+} custom_easy_out_data_s;
+
+/**
+ * @brief Invoke callback to check the output buffer given to a custom-easy filter.
+ */
+static int
+test_custom_easy_out_cb (const ml_tensors_data_h in, ml_tensors_data_h out, void *user_data)
+{
+  custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data;
+  ml_tensors_info_h out_info = NULL;
+  void *raw_data = NULL;
+  size_t data_size = 0;
+  unsigned int count = 0;
+  guint i;
+
+  G_LOCK (callback_lock);
+  result->invoked++;
+
+  /* the number of the output tensors is fixed to the registered info. */
+  if (ml_tensors_data_get_info (out, &out_info) != ML_ERROR_NONE
+      || ml_tensors_info_get_count (out_info, &count) != ML_ERROR_NONE || count != 1U)
+    result->failed = TRUE;
+  ml_tensors_info_destroy (out_info);
+
+  /* the output buffer is allocated by the framework, fill it in-place. */
+  if (ml_tensors_data_get_tensor_data (out, 0, &raw_data, &data_size) != ML_ERROR_NONE
+      || raw_data == NULL || data_size != TEST_CUSTOM_OUT_SIZE) {
+    result->failed = TRUE;
+  } else {
+    for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++)
+      ((guint8 *) raw_data)[i] = TEST_CUSTOM_OUT_VALUE;
+  }
+  G_UNLOCK (callback_lock);
+
+  return 0;
+}
+
+/**
+ * @brief Sink callback to check the output filled by a custom-easy filter.
+ */
+static void
+test_custom_easy_out_sink_cb (
+    const ml_tensors_data_h data, const ml_tensors_info_h info, void *user_data)
+{
+  custom_easy_out_data_s *result = (custom_easy_out_data_s *) user_data;
+  void *raw_data = NULL;
+  size_t data_size = 0;
+  guint i;
+
+  G_LOCK (callback_lock);
+  result->received++;
+
+  if (ml_tensors_data_get_tensor_data (data, 0, &raw_data, &data_size) != ML_ERROR_NONE
+      || data_size != TEST_CUSTOM_OUT_SIZE) {
+    result->failed = TRUE;
+  } else {
+    for (i = 0; i < TEST_CUSTOM_OUT_SIZE; i++) {
+      if (((guint8 *) raw_data)[i] != TEST_CUSTOM_OUT_VALUE)
+        result->failed = TRUE;
+    }
+  }
+  G_UNLOCK (callback_lock);
+}
+
+/**
+ * @brief Test for the output buffer of a custom-easy filter.
+ * @detail The invoke callback gets the output memory owned by tensor_filter,
+ *         sized by the registered output info. Bindings (e.g. Android JNI) copy
+ *         the result into that memory, so releasing or replacing it is invalid.
+ */
+TEST (nnstreamer_capi_custom, invoke_output_buffer_p)
+{
+  const char test_custom_filter[] = "test-custom-filter-out";
+  ml_pipeline_h pipe;
+  ml_pipeline_src_h src;
+  ml_pipeline_sink_h sink;
+  ml_custom_easy_filter_h custom;
+  ml_tensors_info_h in_info, out_info;
+  ml_tensors_data_h in_data;
+  ml_tensor_dimension in_dim = { 2, 1, 1, 1 };
+  ml_tensor_dimension out_dim = { TEST_CUSTOM_OUT_SIZE, 1, 1, 1 };
+  custom_easy_out_data_s *result;
+  int status;
+  guint i;
+  gchar *pipeline = g_strdup_printf (
+      "appsrc name=srcx ! other/tensor,dimension=(string)2:1:1:1,type=(string)int8,framerate=(fraction)0/1 ! tensor_filter framework=custom-easy model=%s ! tensor_sink name=sinkx",
+      test_custom_filter);
+
+  result = (custom_easy_out_data_s *) g_malloc0 (sizeof (custom_easy_out_data_s));
+
+  ml_tensors_info_create (&in_info);
+  ml_tensors_info_set_count (in_info, 1);
+  ml_tensors_info_set_tensor_type (in_info, 0, ML_TENSOR_TYPE_INT8);
+  ml_tensors_info_set_tensor_dimension (in_info, 0, in_dim);
+
+  ml_tensors_info_create (&out_info);
+  ml_tensors_info_set_count (out_info, 1);
+  ml_tensors_info_set_tensor_type (out_info, 0, ML_TENSOR_TYPE_INT8);
+  ml_tensors_info_set_tensor_dimension (out_info, 0, out_dim);
+
+  status = ml_pipeline_custom_easy_filter_register (test_custom_filter, in_info,
+      out_info, test_custom_easy_out_cb, result, &custom);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_construct (pipeline, NULL, NULL, &pipe);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_sink_register (
+      pipe, "sinkx", test_custom_easy_out_sink_cb, result, &sink);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_src_get_handle (pipe, "srcx", &src);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_start (pipe);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  for (i = 0; i < 5; i++) {
+    status = ml_tensors_data_create (in_info, &in_data);
+    EXPECT_EQ (status, ML_ERROR_NONE);
+
+    status = ml_pipeline_src_input_data (src, in_data, ML_PIPELINE_BUF_POLICY_AUTO_FREE);
+    EXPECT_EQ (status, ML_ERROR_NONE);
+
+    g_usleep (50000); /* 50ms. Wait a bit. */
+  }
+
+  status = ml_pipeline_stop (pipe);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_src_release_handle (src);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_sink_unregister (sink);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_destroy (pipe);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  status = ml_pipeline_custom_easy_filter_unregister (custom);
+  EXPECT_EQ (status, ML_ERROR_NONE);
+
+  EXPECT_TRUE (result->invoked > 0U);
+  EXPECT_TRUE (result->received > 0U);
+  EXPECT_FALSE (result->failed);
+
+  ml_tensors_info_destroy (in_info);
+  ml_tensors_info_destroy (out_info);
+  g_free (pipeline);
+  g_free (result);
+}
+
 /**
  * @brief Test for custom-easy registration.
  * @detail Invalid params.